From 0429e897068400de237f8e54023a26903ff82d3e Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 01:44:22 +0000 Subject: [PATCH 01/16] PYTHON-2943 Add SOCKS5 proxy support Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .evergreen/remove-unimplemented-tests.sh | 1 - doc/changelog.rst | 7 + pymongo/asynchronous/encryption.py | 3 + pymongo/asynchronous/mongo_client.py | 11 + pymongo/client_options.py | 14 +- pymongo/common.py | 44 +++ pymongo/pool_options.py | 19 +- pymongo/pool_shared.py | 322 ++++++++++++++++- pymongo/synchronous/encryption.py | 3 + pymongo/synchronous/mongo_client.py | 11 + pymongo/topology_shared.py | 1 + pymongo/uri_parser_shared.py | 35 +- test/asynchronous/test_socks5.py | 430 +++++++++++++++++++++++ test/asynchronous/test_socks5_prose.py | 311 ++++++++++++++++ test/test_socks5.py | 430 +++++++++++++++++++++++ test/test_socks5_prose.py | 311 ++++++++++++++++ test/test_uri_parser.py | 57 +++ test/uri_options/proxy-options.json | 139 ++++++++ 18 files changed, 2143 insertions(+), 6 deletions(-) create mode 100644 test/asynchronous/test_socks5.py create mode 100644 test/asynchronous/test_socks5_prose.py create mode 100644 test/test_socks5.py create mode 100644 test/test_socks5_prose.py create mode 100644 test/uri_options/proxy-options.json diff --git a/.evergreen/remove-unimplemented-tests.sh b/.evergreen/remove-unimplemented-tests.sh index 7982dd6b21..7b2b3b02eb 100755 --- a/.evergreen/remove-unimplemented-tests.sh +++ b/.evergreen/remove-unimplemented-tests.sh @@ -47,7 +47,6 @@ rm $PYMONGO/test/csot/override-collection-timeoutMS.json rm $PYMONGO/test/csot/override-database-timeoutMS.json # PYTHON-2943 - Socks5 Proxy Support -rm $PYMONGO/test/uri_options/proxy-options.json # PYTHON-5517 - Avoid clearing the connection pool when the server connection rate limiter triggers rm $PYMONGO/test/discovery_and_monitoring/unified/backpressure-*.json diff --git a/doc/changelog.rst b/doc/changelog.rst index dc0e697fdf..c4ad313ca5 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -4,6 +4,12 @@ Changelog Changes in Version 4.20.0 (2026/XX/XX) -------------------------------------- +- Added support for connecting to MongoDB and Client-Side Field Level + Encryption KMS servers through a SOCKS5 proxy via the ``proxyHost``, + ``proxyPort``, ``proxyUsername``, and ``proxyPassword`` URI and keyword + options on :class:`~pymongo.synchronous.mongo_client.MongoClient` and + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` + (`PYTHON-2943`_). - The client handshake metadata now reports a detected coding agent in ``client.env.agent`` (`PYTHON-5929`_). @@ -35,6 +41,7 @@ Bug fixes The bug was fixed in Python 3.12.7 and Python 3.13.1. Users of Python 3.12 and 3.13 should upgrade to at least those versions (`PYTHON-6182`_). +.. _PYTHON-2943: https://jira.mongodb.org/browse/PYTHON-2943 .. _PYTHON-5929: https://jira.mongodb.org/browse/PYTHON-5929 .. _PYTHON-6172: https://jira.mongodb.org/browse/PYTHON-6172 .. _PYTHON-6149: https://jira.mongodb.org/browse/PYTHON-6149 diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index e4f91294a4..adfc3e3abd 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -140,6 +140,8 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client + proxy_client = client if client is not None else key_vault_coll.database.client + self._proxy_options = proxy_client.options.pool_options._proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) @@ -182,6 +184,7 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: connect_timeout=connect_timeout, socket_timeout=connect_timeout, ssl_context=ctx, + proxy_options=self._proxy_options, ) try: conn = await _connect_kms( diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index df390a218e..66f0c58f16 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -331,6 +331,14 @@ def __init__( connecting a new socket to a server before concluding the server is unavailable. ``0`` or ``None`` means no timeout. Defaults to ``20000`` (20 seconds). + - `proxyHost` (string): The hostname or IP address of a SOCKS5 proxy. + IPv6 literals may be enclosed in brackets. Defaults to no proxy. + - `proxyPort` (integer): The TCP port of the SOCKS5 proxy. Defaults + to ``1080`` when ``proxyHost`` is specified. + - `proxyUsername` (string): The username for SOCKS5 username/password + authentication. Must be specified together with ``proxyPassword``. + - `proxyPassword` (string): The password for SOCKS5 username/password + authentication. Must be specified together with ``proxyUsername``. - `server_selector`: (callable or None) Optional, user-provided function that augments server selection rules. The function should accept as an argument a list of @@ -1113,6 +1121,9 @@ def _validate_kwargs_and_update_opts( keyword_opts = common._CaseInsensitiveDictionary( dict(common.validate(keyword_opts.cased_key(k), v) for k, v in keyword_opts.items()) ) + for name in ("proxyusername", "proxypassword"): + if keyword_opts.get(name) == "": + keyword_opts.pop(name) # Override connection string options with kwarg options. opts.update(keyword_opts) return opts diff --git a/pymongo/client_options.py b/pymongo/client_options.py index b42635ef2c..6fffa30ccf 100644 --- a/pymongo/client_options.py +++ b/pymongo/client_options.py @@ -27,7 +27,7 @@ from pymongo.compression_support import CompressionSettings from pymongo.errors import ConfigurationError from pymongo.monitoring import _EventListener, _EventListeners -from pymongo.pool_options import PoolOptions +from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ( _ServerMode, @@ -174,6 +174,17 @@ def _parse_pool_options( options.get("compressors", []), options.get("zlibcompressionlevel", -1) ) ssl_context, tls_allow_invalid_hostnames = _parse_ssl_options(options, is_sync) + proxy_host = options.get("proxyhost") + proxy_options = ( + _ProxyOptions( + proxy_host, + options.get("proxyport") or 1080, + options.get("proxyusername"), + options.get("proxypassword"), + ) + if proxy_host is not None + else None + ) load_balanced = options.get("loadbalanced") max_connecting = options.get("maxconnecting", common.MAX_CONNECTING) return PoolOptions( @@ -194,6 +205,7 @@ def _parse_pool_options( load_balanced=load_balanced, credentials=credentials, is_sync=is_sync, + proxy_options=proxy_options, ) diff --git a/pymongo/common.py b/pymongo/common.py index ce4379b507..caa1ad5aad 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -18,6 +18,8 @@ from __future__ import annotations import datetime +import ipaddress +import unicodedata import warnings from collections import OrderedDict, abc from collections.abc import Iterator, Mapping, MutableMapping, Sequence @@ -228,6 +230,44 @@ def validate_non_negative_integer(option: str, value: Any) -> int: return val +def validate_proxy_host(option: str, value: Any) -> str: + """Validate and normalize a SOCKS5 proxy host.""" + host = validate_string(option, value) + if not host: + raise ValueError(f"{option} cannot be an empty string") + if any(char.isspace() or unicodedata.category(char) == "Cc" for char in host): + raise ValueError(f"{option} must not contain whitespace or control characters") + if host.startswith("[") or host.endswith("]"): + if not (host.startswith("[") and host.endswith("]")): + raise ValueError(f"{option} has invalid bracket syntax") + literal = host[1:-1] + try: + ipaddress.IPv6Address(literal.split("%", 1)[0]) + except ipaddress.AddressValueError: + raise ValueError(f"{option} must contain a bracketed IPv6 address") from None + return literal + if "[" in host or "]" in host or "/" in host: + raise ValueError(f"{option} has invalid host syntax") + if ":" in host: + try: + ipaddress.IPv6Address(host.split("%", 1)[0]) + except ipaddress.AddressValueError: + raise ValueError(f"{option} must not include a port") from None + return host + + +def validate_proxy_port(option: str, value: Any) -> Optional[int]: + """Validate a SOCKS5 proxy TCP port.""" + if value is None: + return None + if isinstance(value, bool): + raise TypeError(f"Wrong type for {option}, value must be an integer, not bool") + port = validate_integer(option, value) + if not 1 <= port <= 65535: + raise ValueError(f"{option} must be an integer between 1 and 65535") + return port + + def validate_readable(option: str, value: Any) -> Optional[str]: """Validates that 'value' is file-like and readable.""" if value is None: @@ -725,6 +765,10 @@ def validate_server_monitoring_mode(option: str, value: str) -> str: "maxconnecting": validate_positive_integer, "maxpoolsize": validate_non_negative_integer_or_none, "maxstalenessseconds": validate_max_staleness, + "proxyhost": validate_proxy_host, + "proxyport": validate_proxy_port, + "proxyusername": validate_string_or_none, + "proxypassword": validate_string_or_none, "readconcernlevel": validate_string_or_none, "readpreference": validate_read_preference_mode, "readpreferencetags": validate_read_preference_tags, diff --git a/pymongo/pool_options.py b/pymongo/pool_options.py index 45a9adf5da..e8ab729ea0 100644 --- a/pymongo/pool_options.py +++ b/pymongo/pool_options.py @@ -26,7 +26,7 @@ from collections.abc import MutableMapping from contextlib import AbstractContextManager, nullcontext from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, NamedTuple, Optional import bson from pymongo import __version__ @@ -369,6 +369,15 @@ def _truncate_metadata(metadata: MutableMapping[str, Any]) -> None: "foo".encode("idna") +class _ProxyOptions(NamedTuple): + """Immutable SOCKS5 proxy connection settings.""" + + host: str + port: int = 1080 + username: Optional[str] = None + password: Optional[str] = None + + class PoolOptions: """Read only connection pool options for an AsyncMongoClient/MongoClient. @@ -398,6 +407,7 @@ class PoolOptions: "__metadata_lock", "__min_pool_size", "__pause_enabled", + "__proxy_options", "__server_api", "__socket_timeout", "__ssl_context", @@ -425,6 +435,7 @@ def __init__( load_balanced: Optional[bool] = None, credentials: Optional[MongoCredential] = None, is_sync: Optional[bool] = True, + proxy_options: Optional[_ProxyOptions] = None, ): self.__max_pool_size = max_pool_size self.__min_pool_size = min_pool_size @@ -443,6 +454,7 @@ def __init__( self.__server_api = server_api self.__load_balanced = load_balanced self.__credentials = credentials + self.__proxy_options = proxy_options self.__metadata = copy.deepcopy(_METADATA) self.__appended_drivers: set[DriverInfo] = set() # Only the synchronous client can append metadata from multiple threads. @@ -607,6 +619,11 @@ def wait_queue_timeout(self) -> Optional[int]: """ return self.__wait_queue_timeout + @property + def _proxy_options(self) -> Optional[_ProxyOptions]: + """SOCKS5 proxy settings or None.""" + return self.__proxy_options + @property def _ssl_context(self) -> Optional[SSLContext]: """An SSLContext instance or None.""" diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 48a0a27bf1..19f8c23ea8 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -20,6 +20,7 @@ import collections import contextlib import functools +import ipaddress import os import socket import ssl @@ -63,7 +64,7 @@ NetworkingInterfaceBase, PyMongoProtocol, ) -from pymongo.pool_options import PoolOptions +from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.server_api import _add_to_command from pymongo.socket_checker import SocketChecker from pymongo.ssl_support import SSL_EOF_ERRORS, PYSSLError, SSLError, _has_sni @@ -341,7 +342,10 @@ def _handle_connection_error(self, error: BaseException) -> None: return assert isinstance(error, AutoReconnect) # Appease type checker. # If the original error was a DNS, certificate, or SSL error, ignore it. - if isinstance(error.__cause__, (_CertificateError, SSLErrors, socket.gaierror)): + if isinstance( + error.__cause__, + (_CertificateError, SSLErrors, socket.gaierror, _Socks5Error, _Socks5Timeout), + ): # End of file errors are excluded, because the server may have disconnected # during the handshake. if not isinstance(error.__cause__, SSL_EOF_ERRORS): @@ -593,6 +597,314 @@ class PoolState: CLOSED = 3 +class _Socks5Error(OSError): + """A SOCKS5 proxy connection or protocol failure.""" + + +class _Socks5Timeout(socket.timeout): + """A timeout while connecting through a SOCKS5 proxy.""" + + +def _proxy_deadline(options: PoolOptions) -> Optional[float]: + timeout = _csot.remaining() + if timeout is None: + timeout = options.connect_timeout + elif timeout <= 0: + raise _Socks5Timeout("timed out") + if timeout is None: + return None + return time.monotonic() + timeout + + +def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: + if deadline is None: + return None + remaining = deadline - time.monotonic() + if remaining <= 0: + raise _Socks5Timeout("timed out") + return remaining + + +def _socks5_request(address: _Address) -> bytes: + host, port = address + try: + ip = ipaddress.ip_address(host.split("%", 1)[0]) + except ValueError: + try: + domain = host.encode("idna") + except UnicodeError as exc: + raise _Socks5Error(f"SOCKS5 connect failed: invalid domain name: {exc}") from exc + if not 1 <= len(domain) <= 255: + raise _Socks5Error("SOCKS5 connect failed: invalid domain name length") from None + encoded_host = b"\x03" + bytes((len(domain),)) + domain + else: + encoded_host = (b"\x01" if ip.version == 4 else b"\x04") + ip.packed + if port is None: + raise _Socks5Error("SOCKS5 connect failed: destination port is required") + return b"\x05\x01\x00" + encoded_host + port.to_bytes(2, "big") + + +def _socks5_read_exact(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: + data = bytearray() + while len(data) < size: + sock.settimeout(_proxy_remaining(deadline)) + chunk = sock.recv(size - len(data)) + if not chunk: + raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") + data.extend(chunk) + return bytes(data) + + +def _socks5_handshake( + sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] +) -> None: + has_credentials = proxy.username is not None and proxy.password is not None + greeting = b"\x05\x02\x00\x02" if has_credentials else b"\x05\x01\x00" + sock.settimeout(_proxy_remaining(deadline)) + sock.sendall(greeting) + version, method = _socks5_read_exact(sock, 2, deadline) + if version != 0x05: + raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") + allowed = (0x00, 0x02) if has_credentials else (0x00,) + if method not in allowed: + if method == 0xFF: + raise _Socks5Error( + "SOCKS5 initial handshake failed: no acceptable authentication methods" + ) + raise _Socks5Error( + f"SOCKS5 initial handshake failed: unexpected authentication method {method:#04x}" + ) + if method == 0x02: + assert proxy.username is not None + assert proxy.password is not None + username = proxy.username.encode("utf-8") + password = proxy.password.encode("utf-8") + sock.settimeout(_proxy_remaining(deadline)) + sock.sendall(bytes((0x01, len(username))) + username + bytes((len(password),)) + password) + auth_version, status = _socks5_read_exact(sock, 2, deadline) + if auth_version != 0x01 or status != 0x00: + raise _Socks5Error("SOCKS5 authentication failed") + + sock.settimeout(_proxy_remaining(deadline)) + sock.sendall(_socks5_request(address)) + version, reply, reserved, atyp = _socks5_read_exact(sock, 4, deadline) + if version != 0x05: + raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") + if reserved != 0x00: + raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") + if reply: + reasons = { + 0x01: "general SOCKS server failure", + 0x02: "connection not allowed by ruleset", + 0x03: "network unreachable", + 0x04: "host unreachable", + 0x05: "connection refused", + 0x06: "TTL expired", + 0x07: "command not supported", + 0x08: "address type not supported", + } + reason = reasons.get(reply, f"unknown error {reply:#04x}") + raise _Socks5Error(f"SOCKS5 connect failed: {reason}") + if atyp == 0x01: + bound_size = 6 + elif atyp == 0x04: + bound_size = 18 + elif atyp == 0x03: + domain_length = _socks5_read_exact(sock, 1, deadline)[0] + bound_size = domain_length + 2 + else: + raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") + _socks5_read_exact(sock, bound_size, deadline) + + +async def _async_socks5_read_exact( + sock: socket.socket, size: int, deadline: Optional[float] +) -> bytes: + data = bytearray() + loop = asyncio.get_running_loop() + while len(data) < size: + chunk = await asyncio.wait_for( + loop.sock_recv(sock, size - len(data)), timeout=_proxy_remaining(deadline) + ) + if not chunk: + raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") + data.extend(chunk) + return bytes(data) + + +async def _async_socks5_handshake( + sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] +) -> None: + loop = asyncio.get_running_loop() + has_credentials = proxy.username is not None and proxy.password is not None + greeting = b"\x05\x02\x00\x02" if has_credentials else b"\x05\x01\x00" + await asyncio.wait_for(loop.sock_sendall(sock, greeting), timeout=_proxy_remaining(deadline)) + version, method = await _async_socks5_read_exact(sock, 2, deadline) + if version != 0x05: + raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") + allowed = (0x00, 0x02) if has_credentials else (0x00,) + if method not in allowed: + if method == 0xFF: + raise _Socks5Error( + "SOCKS5 initial handshake failed: no acceptable authentication methods" + ) + raise _Socks5Error( + f"SOCKS5 initial handshake failed: unexpected authentication method {method:#04x}" + ) + if method == 0x02: + assert proxy.username is not None + assert proxy.password is not None + username = proxy.username.encode("utf-8") + password = proxy.password.encode("utf-8") + auth = bytes((0x01, len(username))) + username + bytes((len(password),)) + password + await asyncio.wait_for(loop.sock_sendall(sock, auth), timeout=_proxy_remaining(deadline)) + auth_version, status = await _async_socks5_read_exact(sock, 2, deadline) + if auth_version != 0x01 or status != 0x00: + raise _Socks5Error("SOCKS5 authentication failed") + await asyncio.wait_for( + loop.sock_sendall(sock, _socks5_request(address)), timeout=_proxy_remaining(deadline) + ) + version, reply, reserved, atyp = await _async_socks5_read_exact(sock, 4, deadline) + if version != 0x05: + raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") + if reserved != 0x00: + raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") + if reply: + reasons = { + 0x01: "general SOCKS server failure", + 0x02: "connection not allowed by ruleset", + 0x03: "network unreachable", + 0x04: "host unreachable", + 0x05: "connection refused", + 0x06: "TTL expired", + 0x07: "command not supported", + 0x08: "address type not supported", + } + reason = reasons.get(reply, f"unknown error {reply:#04x}") + raise _Socks5Error(f"SOCKS5 connect failed: {reason}") + if atyp == 0x01: + bound_size = 6 + elif atyp == 0x04: + bound_size = 18 + elif atyp == 0x03: + domain_length = (await _async_socks5_read_exact(sock, 1, deadline))[0] + bound_size = domain_length + 2 + else: + raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") + await _async_socks5_read_exact(sock, bound_size, deadline) + + +async def _async_create_proxy_connection( + address: _Address, options: PoolOptions, proxy: _ProxyOptions +) -> socket.socket: + deadline = _proxy_deadline(options) + family = socket.AF_INET + if socket.has_ipv6 and proxy.host != "localhost": + family = socket.AF_UNSPEC + try: + getaddrinfo = _getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) + results = await asyncio.wait_for(getaddrinfo, timeout=_proxy_remaining(deadline)) + except asyncio.TimeoutError as exc: + raise _Socks5Timeout("timed out") from exc + except OSError as exc: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc + + err: Optional[OSError] = None + loop = asyncio.get_running_loop() + for af, socktype, proto, _, sockaddr in results: + try: + sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) + except OSError: + sock = socket.socket(af, socktype, proto) + _set_non_inheritable_non_atomic(sock.fileno()) + connected = False + try: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + _set_keepalive_times(sock) + sock.setblocking(False) + await asyncio.wait_for( + loop.sock_connect(sock, sockaddr), timeout=_proxy_remaining(deadline) + ) + connected = True + await _async_socks5_handshake(sock, address, proxy, deadline) + sock.settimeout(_proxy_remaining(deadline)) + return sock + except asyncio.TimeoutError as exc: + sock.close() + raise _Socks5Timeout("timed out") from exc + except asyncio.CancelledError: + sock.close() + raise + except OSError as exc: + sock.close() + if isinstance(exc, _Socks5Error): + raise + if isinstance(exc, socket.timeout): + raise _Socks5Timeout("timed out") from exc + err = exc + # A connected proxy's handshake failure must not try a different + # resolved proxy address; the failure is specific to the endpoint. + if connected: + raise _Socks5Error(f"SOCKS5 handshake failed: {exc}") from exc + except BaseException: + sock.close() + raise + + if err is not None: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err + raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") + + +def _create_proxy_connection( + address: _Address, options: PoolOptions, proxy: _ProxyOptions +) -> socket.socket: + deadline = _proxy_deadline(options) + family = socket.AF_INET + if socket.has_ipv6 and proxy.host != "localhost": + family = socket.AF_UNSPEC + try: + results = socket.getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) + except OSError as exc: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc + + err: Optional[OSError] = None + for af, socktype, proto, _, sockaddr in results: + try: + sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) + except OSError: + sock = socket.socket(af, socktype, proto) + _set_non_inheritable_non_atomic(sock.fileno()) + connected = False + try: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + _set_keepalive_times(sock) + sock.settimeout(_proxy_remaining(deadline)) + sock.connect(sockaddr) + connected = True + _socks5_handshake(sock, address, proxy, deadline) + return sock + except socket.timeout as exc: + sock.close() + raise _Socks5Timeout("timed out") from exc + except OSError as exc: + sock.close() + if isinstance(exc, _Socks5Error): + raise + err = exc + if connected: + # Handshake failures are not address-resolution failures. + raise _Socks5Error(f"SOCKS5 handshake failed: {exc}") from exc + except BaseException: + sock.close() + raise + + if err is not None: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err + raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") + + async def _async_create_connection(address: _Address, options: PoolOptions) -> socket.socket: """Given (host, port) and PoolOptions, connect and return a raw socket object. @@ -602,6 +914,9 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s """ host, port = address + if options._proxy_options is not None: + return await _async_create_proxy_connection(address, options, options._proxy_options) + # Check if dealing with a unix domain socket if host.endswith(".sock"): if not hasattr(socket, "AF_UNIX"): @@ -849,6 +1164,9 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket """ host, port = address + if options._proxy_options is not None: + return _create_proxy_connection(address, options, options._proxy_options) + # Check if dealing with a unix domain socket if host.endswith(".sock"): if not hasattr(socket, "AF_UNIX"): diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 1626c16f58..827d69e507 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -139,6 +139,8 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client + proxy_client = client if client is not None else key_vault_coll.database.client + self._proxy_options = proxy_client.options.pool_options._proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) @@ -181,6 +183,7 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: connect_timeout=connect_timeout, socket_timeout=connect_timeout, ssl_context=ctx, + proxy_options=self._proxy_options, ) try: conn = _connect_kms( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 6aaaaed20a..981350db59 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -332,6 +332,14 @@ def __init__( connecting a new socket to a server before concluding the server is unavailable. ``0`` or ``None`` means no timeout. Defaults to ``20000`` (20 seconds). + - `proxyHost` (string): The hostname or IP address of a SOCKS5 proxy. + IPv6 literals may be enclosed in brackets. Defaults to no proxy. + - `proxyPort` (integer): The TCP port of the SOCKS5 proxy. Defaults + to ``1080`` when ``proxyHost`` is specified. + - `proxyUsername` (string): The username for SOCKS5 username/password + authentication. Must be specified together with ``proxyPassword``. + - `proxyPassword` (string): The password for SOCKS5 username/password + authentication. Must be specified together with ``proxyUsername``. - `server_selector`: (callable or None) Optional, user-provided function that augments server selection rules. The function should accept as an argument a list of @@ -1114,6 +1122,9 @@ def _validate_kwargs_and_update_opts( keyword_opts = common._CaseInsensitiveDictionary( dict(common.validate(keyword_opts.cased_key(k), v) for k, v in keyword_opts.items()) ) + for name in ("proxyusername", "proxypassword"): + if keyword_opts.get(name) == "": + keyword_opts.pop(name) # Override connection string options with kwarg options. opts.update(keyword_opts) return opts diff --git a/pymongo/topology_shared.py b/pymongo/topology_shared.py index 50bf0c8ad1..8bf9829b94 100644 --- a/pymongo/topology_shared.py +++ b/pymongo/topology_shared.py @@ -262,6 +262,7 @@ def _create_pool_for_monitor(self, address: _Address) -> _PoolT: driver=options.driver, pause_enabled=False, server_api=options.server_api, + proxy_options=options._proxy_options, ) return self._settings.pool_class( diff --git a/pymongo/uri_parser_shared.py b/pymongo/uri_parser_shared.py index 5e34c2d702..b85ab8d5d7 100644 --- a/pymongo/uri_parser_shared.py +++ b/pymongo/uri_parser_shared.py @@ -23,7 +23,7 @@ import re import sys import warnings -from collections.abc import Mapping, MutableMapping, Sized +from collections.abc import Iterable, Mapping, MutableMapping, Sized from typing import ( TYPE_CHECKING, Any, @@ -39,6 +39,7 @@ URI_OPTIONS_DEPRECATION_MAP, _CaseInsensitiveDictionary, get_validated_options, + validate_proxy_host, ) from pymongo.errors import ConfigurationError, InvalidURI from pymongo.typings import _Address @@ -249,6 +250,8 @@ def _parse_options(opts: str, delim: Optional[str]) -> _CaseInsensitiveDictionar options.setdefault(key, []).append(value) else: if key in options: + if key.lower() in ("proxyhost", "proxyport", "proxyusername", "proxypassword"): + raise InvalidURI(f"Duplicate URI option '{key}' is not allowed") warnings.warn(f"Duplicate URI option '{key}'.", stacklevel=2) if key.lower() == "authmechanismproperties": val = value @@ -365,6 +368,10 @@ def _normalize_options(options: _CaseInsensitiveDictionary) -> _CaseInsensitiveD # Implicit options are logically the same as tlsInsecure. options[opt] = tlsinsecure + for optname in ("proxyusername", "proxypassword"): + if options.get(optname) == "": + options.pop(optname) + for optname in list(options): intname = INTERNAL_URI_OPTION_NAME_MAP.get(optname, None) if intname is not None: @@ -492,6 +499,32 @@ def _check_options(nodes: Sized, options: Mapping[str, Any]) -> None: if options.get("replicaset"): raise ConfigurationError("Cannot specify replicaSet with loadBalanced=true") + proxy_host = options.get("proxyhost") + proxy_port = options.get("proxyport") + proxy_username = options.get("proxyusername") + proxy_password = options.get("proxypassword") + if proxy_username == "": + proxy_username = None + if proxy_password == "": + proxy_password = None + if proxy_host is None: + if proxy_port is not None or proxy_username is not None or proxy_password is not None: + raise ConfigurationError( + "Cannot specify proxyPort, proxyUsername, or proxyPassword without proxyHost" + ) + return + + # Validate here as well as in the option validator because this function is + # also used by callers that disable URI option validation. + validate_proxy_host("proxyHost", proxy_host) + if bool(proxy_username) != bool(proxy_password): + raise ConfigurationError("Both proxyUsername and proxyPassword must be specified together") + for name, value in (("proxyUsername", proxy_username), ("proxyPassword", proxy_password)): + if value is not None and len(value.encode("utf-8")) > 255: + raise ConfigurationError(f"{name} must be no more than 255 UTF-8 bytes") + if any(host.endswith(".sock") for host, _ in cast(Iterable[_Address], nodes)): + raise ConfigurationError("Cannot specify proxyHost with Unix domain sockets") + def _parse_kms_tls_options( kms_tls_options: Optional[Mapping[str, Any]], diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py new file mode 100644 index 0000000000..2c86d10e8f --- /dev/null +++ b/test/asynchronous/test_socks5.py @@ -0,0 +1,430 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for SOCKS5 proxy connection establishment.""" + +from __future__ import annotations + +import asyncio +import select +import socket +import ssl +import sys +import threading +from types import SimpleNamespace +from typing import Any, Optional, cast +from unittest import mock + +import pytest + +sys.path[0:0] = [""] + +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT +from pymongo.pool_options import PoolOptions, _ProxyOptions +from pymongo.pool_shared import _async_configured_socket, _async_create_connection, _Socks5Error +from test.asynchronous import AsyncPyMongoTestCase, unittest +from test.helpers_shared import CERT_PATH + +_IS_SYNC = False +pytestmark = pytest.mark.default_async + + +class _Socks5Server: + def __init__( + self, + method: int = 0, + auth_status: int = 0, + reply: int = 0, + bound_atyp: int = 1, + payload: bytes = b"", + method_version: int = 5, + reply_version: int = 5, + eof_stage: Optional[str] = None, + stall: bool = False, + target: Optional[tuple[str, int]] = None, + ) -> None: + self.method = method + self.auth_status = auth_status + self.reply = reply + self.bound_atyp = bound_atyp + self.payload = payload + self.method_version = method_version + self.reply_version = reply_version + self.eof_stage = eof_stage + self.stall = stall + self.target = target + self.started = threading.Event() + self.greeting = b"" + self.auth = b"" + self.request = b"" + self.error: Optional[BaseException] = None + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.listener.settimeout(3) + self.address = self.listener.getsockname() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + @staticmethod + def _read(sock: socket.socket, size: int) -> bytes: + data = bytearray() + while len(data) < size: + part = sock.recv(size - len(data)) + if not part: + raise EOFError("client closed connection") + data.extend(part) + return bytes(data) + + def _serve(self) -> None: + upstream: Optional[socket.socket] = None + try: + with self.listener.accept()[0] as conn: + conn.settimeout(3) + version, nmethods = self._read(conn, 2) + self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) + self.started.set() + if self.stall: + conn.recv(1) + return + if self.eof_stage == "greeting": + conn.sendall(b"\x05") + return + conn.sendall(bytes((self.method_version, self.method))) + if self.method == 2: + auth_version, username_length = self._read(conn, 2) + username = self._read(conn, username_length) + password_length = self._read(conn, 1)[0] + password = self._read(conn, password_length) + self.auth = ( + bytes((auth_version, username_length)) + + username + + bytes((password_length,)) + + password + ) + if self.eof_stage == "auth": + conn.sendall(b"\x01") + return + conn.sendall(bytes((1, self.auth_status))) + if self.auth_status: + return + version, command, reserved, atyp = self._read(conn, 4) + if atyp == 1: + addr = self._read(conn, 4) + elif atyp == 4: + addr = self._read(conn, 16) + else: + length = self._read(conn, 1) + addr = length + self._read(conn, length[0]) + self.request = ( + bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) + ) + if self.eof_stage == "connect": + conn.sendall(b"\x05\x00") + return + if self.target is not None: + upstream = socket.create_connection(self.target, timeout=3) + if self.bound_atyp == 1: + bound = b"\x01\x7f\x00\x00\x01\x00\x00" + elif self.bound_atyp == 4: + bound = b"\x04" + (b"\x00" * 16) + b"\x00\x00" + else: + bound = b"\x03\x01x\x00\x00" + conn.sendall(bytes((self.reply_version, self.reply, 0)) + bound + self.payload) + if self.payload: + return + if upstream is not None: + conn.setblocking(False) + upstream.setblocking(False) + while True: + readable, _, _ = select.select([conn, upstream], [], [], 5) + if not readable: + return + for source in readable: + data = source.recv(65536) + if not data: + return + destination = upstream if source is conn else conn + destination.sendall(data) + else: + while conn.recv(1024): + pass + except BaseException as exc: + self.error = exc + finally: + if upstream is not None: + upstream.close() + self.listener.close() + + def close(self) -> None: + self.listener.close() + self.thread.join(timeout=3) + + +class TestSocks5(AsyncPyMongoTestCase): + async def _connect( + self, + server: _Socks5Server, + address: tuple[str, int], + username: Optional[str] = None, + password: Optional[str] = None, + timeout: float = 2, + ) -> socket.socket: + return await _async_create_connection( + address, + PoolOptions( + connect_timeout=timeout, + proxy_options=_ProxyOptions( + server.address[0], server.address[1], username, password + ), + ), + ) + + async def test_greeting_and_authenticated_connect(self) -> None: + server = _Socks5Server(method=2) + sock = await self._connect(server, ("mongodb.example", 27017), "üser", "påss") + sock.close() + server.close() + self.assertEqual(server.greeting, b"\x05\x02\x00\x02") + self.assertEqual(server.auth, b"\x01\x05" + "üser".encode() + b"\x05" + "påss".encode()) + self.assertEqual(server.request[:4], b"\x05\x01\x00\x03") + self.assertEqual( + server.request[4:], + bytes((len(b"mongodb.example"),)) + b"mongodb.example" + (27017).to_bytes(2, "big"), + ) + + async def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: + server = _Socks5Server(method=0) + sock = await self._connect(server, ("127.0.0.1", 27017), "user", "password") + sock.close() + server.close() + self.assertEqual(server.greeting, b"\x05\x02\x00\x02") + self.assertEqual(server.auth, b"") + self.assertEqual(server.request, b"\x05\x01\x00\x01\x7f\x00\x00\x01\x69\x89") + + async def test_no_credentials_only_offer_no_authentication(self) -> None: + server = _Socks5Server(method=2) + with self.assertRaises(_Socks5Error): + await self._connect(server, ("localhost", 27017)) + server.close() + self.assertEqual(server.greeting, b"\x05\x01\x00") + + async def test_rejects_all_unacceptable_methods(self) -> None: + for method in (0x01, 0xFF): + server = _Socks5Server(method=method) + with self.subTest(method=method), self.assertRaises(_Socks5Error): + await self._connect(server, ("localhost", 27017)) + server.close() + + async def test_rejects_authentication_failure(self) -> None: + server = _Socks5Server(method=2, auth_status=1) + with self.assertRaises(_Socks5Error): + await self._connect(server, ("localhost", 27017), "user", "password") + server.close() + self.assertEqual(server.auth, b"\x01\x04user\x08password") + + async def test_connect_request_address_types(self) -> None: + cases = ( + ("192.0.2.1", b"\x01\xc0\x00\x02\x01"), + ("2001:db8::1", b"\x04" + bytes.fromhex("20010db8000000000000000000000001")), + ( + "münich.example", + b"\x03" + + bytes((len("xn--mnich-kva.example"),)) + + "xn--mnich-kva.example".encode("ascii"), + ), + ) + for host, expected in cases: + server = _Socks5Server() + sock = await self._connect(server, (host, 27017)) + sock.close() + server.close() + self.assertEqual(server.request, b"\x05\x01\x00" + expected + b"\x69\x89") + + async def test_drains_bound_address_and_preserves_early_payload(self) -> None: + for atyp in (1, 3, 4): + server = _Socks5Server(bound_atyp=atyp, payload=b"mongo") + sock = await self._connect(server, ("localhost", 27017)) + if _IS_SYNC: + payload = sock.recv(5) + else: + sock.setblocking(False) + payload = await asyncio.get_running_loop().sock_recv(sock, 5) + sock.close() + server.close() + self.assertEqual(payload, b"mongo") + + async def test_rejects_socks_reply_codes(self) -> None: + for reply in range(1, 9): + server = _Socks5Server(reply=reply) + with self.subTest(reply=reply), self.assertRaises(_Socks5Error): + await self._connect(server, ("localhost", 27017)) + server.close() + + async def test_proxy_options_reach_server_and_monitor_pools(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + from pymongo.asynchronous.topology import Topology + + client = AsyncMongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) + try: + topology = Topology(client._topology_settings) + server_pool = topology._create_pool_for_server(("localhost", 27017)) + monitor_pool = topology._create_pool_for_monitor(("localhost", 27017)) + expected = _ProxyOptions("127.0.0.1") + self.assertEqual(server_pool.opts._proxy_options, expected) + self.assertEqual(monitor_pool.opts._proxy_options, expected) + finally: + await client.close() + + async def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + client = AsyncMongoClient( + "mongodb://localhost/?proxyHost=localhost&proxyPort=1234", + proxyHost="[::1]", + proxyPort=5678, + proxyUsername="", + proxyPassword="", + connect=False, + ) + try: + self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1", 5678)) + finally: + await client.close() + + async def test_bad_versions_and_unexpected_eof_fail(self) -> None: + cases = ( + (_Socks5Server(method_version=4), ("localhost", 27017), None), + (_Socks5Server(reply_version=4), ("localhost", 27017), None), + (_Socks5Server(eof_stage="greeting"), ("localhost", 27017), None), + ( + _Socks5Server(method=2, eof_stage="auth"), + ("localhost", 27017), + ("user", "password"), + ), + (_Socks5Server(eof_stage="connect"), ("localhost", 27017), None), + ) + for server, address, credentials in cases: + with self.subTest(stage=server.eof_stage, version=server.method_version): + with self.assertRaises(_Socks5Error): + await self._connect(server, address, *(credentials or (None, None))) + server.close() + + async def test_timeout_covers_the_entire_handshake(self) -> None: + server = _Socks5Server(stall=True) + with self.assertRaises(socket.timeout): + await self._connect(server, ("localhost", 27017), timeout=0.1) + server.close() + + @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the asynchronous driver") + async def test_cancellation_closes_proxy_socket(self) -> None: + server = _Socks5Server(stall=True) + task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) + await asyncio.to_thread(server.started.wait, 2) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + server.close() + + async def test_tls_sni_targets_destination_not_proxy(self) -> None: + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + target_address = listener.getsockname() + tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") + sni_names: list[Optional[str]] = [] + tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + + def serve_tls() -> None: + try: + raw, _ = listener.accept() + with tls_context.wrap_socket(raw, server_side=True) as conn: + conn.recv(1) + except OSError: + pass + finally: + listener.close() + + tls_thread = threading.Thread(target=serve_tls, daemon=True) + tls_thread.start() + proxy = _Socks5Server(target=target_address) + client_context = ssl.create_default_context() + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + sock = await _async_configured_socket( + ("mongodb.example", target_address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, client_context), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), + ), + ) + sock.close() + proxy.close() + tls_thread.join(timeout=3) + self.assertEqual(sni_names, ["mongodb.example"]) + + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + async def test_kms_request_uses_client_proxy_options(self) -> None: + import pymongo.asynchronous.encryption as encryption_module + from pymongo.asynchronous.encryption import _EncryptionIO + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + class FakeOpts: + _kms_connect_callback = None + + @staticmethod + def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: + return {"aws": object()} + + client = AsyncMongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) + expected = _ProxyOptions("127.0.0.1") + io = _EncryptionIO(client, client.keyvault.datakeys, None, FakeOpts()) # type: ignore[arg-type] + connection = mock.MagicMock() + + async def connect(_address, options, _callback, _timeout): + self.assertEqual(options._proxy_options, expected) + return connection + + async def sendall(_connection, _message): + pass + + try: + with ( + mock.patch.object(encryption_module, "_connect_kms", new=connect), + mock.patch.object(encryption_module, "async_socket_sendall", new=sendall), + ): + await io.kms_request( + SimpleNamespace( + endpoint="kms.example:443", + message=b"request", + kms_provider="aws", + usleep=0, + bytes_needed=0, + ) + ) + finally: + await client.close() + connection.close.assert_called_once_with() + + async def test_proxy_settings_are_stored_in_pool_options(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + client = AsyncMongoClient("mongodb://localhost/?proxyHost=[::1]", connect=False) + try: + self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1")) + finally: + await client.close() diff --git a/test/asynchronous/test_socks5_prose.py b/test/asynchronous/test_socks5_prose.py new file mode 100644 index 0000000000..605bb1f16a --- /dev/null +++ b/test/asynchronous/test_socks5_prose.py @@ -0,0 +1,311 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SOCKS5 connection-string prose tests.""" + +from __future__ import annotations + +import os +import select +import socket +import threading +from typing import Optional +from urllib.parse import quote_plus + +import pytest + +from pymongo import monitoring +from pymongo.asynchronous.mongo_client import AsyncMongoClient +from pymongo.hello import HelloCompat +from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest +from test.helpers_shared import TLS_OPTIONS, db_pwd, db_user + +_IS_SYNC = False +pytestmark = pytest.mark.default_async + + +class _CommandListener(monitoring.CommandListener): + def __init__(self) -> None: + self.connection_ids: list[tuple[str, Optional[int]]] = [] + + def started(self, event: monitoring.CommandStartedEvent) -> None: + self.connection_ids.append(event.connection_id) + + def succeeded(self, event: monitoring.CommandSucceededEvent) -> None: + self.connection_ids.append(event.connection_id) + + def failed(self, event: monitoring.CommandFailedEvent) -> None: + self.connection_ids.append(event.connection_id) + + +class Socks5Server: + """Small threaded SOCKS5 forwarder for the driver prose tests.""" + + def __init__( + self, + port: int, + mapping: dict[tuple[str, int], tuple[str, int]], + credentials: Optional[tuple[str, str]] = None, + ) -> None: + self.mapping = mapping + self.credentials = credentials + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + try: + self.listener.bind(("127.0.0.1", port)) + except OSError: + self.listener.bind(("127.0.0.1", 0)) + self.port = self.listener.getsockname()[1] + self.listener.listen() + self.listener.settimeout(0.5) + self._closed = threading.Event() + self._threads: list[threading.Thread] = [] + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + @staticmethod + def _read(sock: socket.socket, length: int) -> bytes: + data = bytearray() + while len(data) < length: + part = sock.recv(length - len(data)) + if not part: + raise EOFError + data.extend(part) + return bytes(data) + + def _serve_client(self, client: socket.socket) -> None: + upstream: Optional[socket.socket] = None + try: + client.settimeout(5) + version, method_count = self._read(client, 2) + methods = self._read(client, method_count) + if version != 5: + return + if self.credentials is not None: + method = 2 if 2 in methods else 0xFF + else: + method = 0 if 0 in methods else 0xFF + client.sendall(bytes((5, method))) + if method == 0xFF: + return + if method == 2: + version, username_length = self._read(client, 2) + username = self._read(client, username_length) + password_length = self._read(client, 1)[0] + password = self._read(client, password_length) + user, secret = self.credentials or ("", "") + status = ( + 0 + if version == 1 and username == user.encode() and password == secret.encode() + else 1 + ) + client.sendall(bytes((1, status))) + if status: + return + version, command, reserved, atyp = self._read(client, 4) + if atyp == 1: + host = socket.inet_ntop(socket.AF_INET, self._read(client, 4)) + elif atyp == 4: + host = socket.inet_ntop(socket.AF_INET6, self._read(client, 16)) + elif atyp == 3: + length = self._read(client, 1)[0] + host = self._read(client, length).decode("idna") + else: + client.sendall(b"\x05\x08\x00\x01\x00\x00\x00\x00\x00\x00") + return + port = int.from_bytes(self._read(client, 2), "big") + if version != 5 or command != 1 or reserved != 0: + client.sendall(b"\x05\x07\x00\x01\x00\x00\x00\x00\x00\x00") + return + target = self.mapping.get((host.lower(), port), (host, port)) + try: + upstream = socket.create_connection(target, timeout=5) + except OSError: + client.sendall(b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00") + return + bound_host, bound_port = upstream.getsockname()[:2] + try: + bound_ip = socket.inet_pton(socket.AF_INET, bound_host) + atyp = 1 + except OSError: + bound_ip = socket.inet_pton(socket.AF_INET6, bound_host) + atyp = 4 + client.sendall(bytes((5, 0, 0, atyp)) + bound_ip + int(bound_port).to_bytes(2, "big")) + client.setblocking(False) + upstream.setblocking(False) + while True: + readable, _, _ = select.select([client, upstream], [], [], 10) + if not readable: + continue + for source in readable: + data = source.recv(65536) + if not data: + return + target_sock = upstream if source is client else client + target_sock.sendall(data) + except (OSError, EOFError): + pass + finally: + if upstream is not None: + upstream.close() + client.close() + + def _serve(self) -> None: + while not self._closed.is_set(): + try: + client, _ = self.listener.accept() + except socket.timeout: + continue + except OSError: + break + thread = threading.Thread(target=self._serve_client, args=(client,), daemon=True) + self._threads.append(thread) + thread.start() + + def close(self) -> None: + self._closed.set() + self.listener.close() + self._thread.join(timeout=2) + for thread in self._threads: + thread.join(timeout=2) + + +class TestSocks5Prose(AsyncIntegrationTest): + @async_client_context.require_replica_set + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + self.nodes = sorted(async_client_context.nodes) + self.mapping = {("localhost", 12345): self.nodes[0]} + self.credentials = getattr(self, "credentials", {}) + if os.environ.get("SOCKS5_PROXIES_STARTED"): + self.auth_port = 1080 + self.noauth_port = 1081 + else: + auth_proxy = Socks5Server(1080, self.mapping, ("username", "p4ssw0rd")) + noauth_proxy = Socks5Server(1081, self.mapping) + self.addCleanup(auth_proxy.close) + self.addCleanup(noauth_proxy.close) + self.auth_port = auth_proxy.port + self.noauth_port = noauth_proxy.port + + def _hosts(self, mapped: bool) -> str: + if mapped: + return "localhost:12345" + return ",".join( + f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" + for host, port in self.nodes + ) + + async def _run_case( + self, + hosts: str, + port: int, + credentials: Optional[tuple[str, str]], + succeeds: bool, + use_uri: bool, + listener: Optional[_CommandListener] = None, + ) -> None: + proxy_options = {"proxyHost": "localhost", "proxyPort": port} + if credentials: + proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials + options = dict(TLS_OPTIONS) + options.update(self.credentials) + options.update(proxy_options) + options["serverSelectionTimeoutMS"] = 2500 + if hosts == "localhost:12345": + options["directConnection"] = True + if listener is not None: + options["event_listeners"] = [listener] + if use_uri: + uri_options = "&".join( + f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() + ) + uri_options += "&serverSelectionTimeoutMS=2500" + if hosts == "localhost:12345": + uri_options += "&directConnection=true" + if TLS_OPTIONS: + uri_options += "&" + "&".join( + f"{key}={quote_plus(str(value))}" for key, value in TLS_OPTIONS.items() + ) + auth = "" + if self.credentials: + auth = f"{quote_plus(self.credentials['username'])}:{quote_plus(self.credentials['password'])}@" + client = AsyncMongoClient( + f"mongodb://{auth}{hosts}/?{uri_options}", + event_listeners=options.get("event_listeners"), + ) + else: + client = AsyncMongoClient(hosts, **options) + try: + if succeeds: + await client.admin.command(HelloCompat.CMD) + if listener is not None: + from pymongo.errors import OperationFailure + + with self.assertRaises(OperationFailure): + await client.admin.command("nonexistent_socks5_test_command") + else: + from pymongo.errors import ConnectionFailure + + with self.assertRaises(ConnectionFailure): + await client.admin.command(HelloCompat.CMD) + finally: + await client.close() + + async def test_connection_string_prose_table(self) -> None: + cases = ( + (True, self.auth_port, None, False), + (True, self.noauth_port, None, True), + (False, self.auth_port, None, False), + (False, self.noauth_port, None, True), + (True, self.auth_port, ("nonexistentuser", "badauth"), False), + (True, self.noauth_port, ("nonexistentuser", "badauth"), True), + (False, self.noauth_port, ("nonexistentuser", "badauth"), True), + (True, self.auth_port, ("username", "p4ssw0rd"), True), + (True, self.noauth_port, None, True), + (False, self.auth_port, ("username", "p4ssw0rd"), True), + (False, self.noauth_port, None, True), + ) + mapped_listener = _CommandListener() + rs_listener = _CommandListener() + for index, (mapped, proxy_port, credentials, succeeds) in enumerate(cases): + for use_uri in (True, False): + listener = None + if not use_uri: + if index == 1: + listener = mapped_listener + elif index == 3: + listener = rs_listener + await self._run_case( + self._hosts(mapped), + proxy_port, + credentials, + succeeds, + use_uri, + listener, + ) + proxy_addresses = { + ("localhost", self.auth_port), + ("localhost", self.noauth_port), + ("127.0.0.1", self.auth_port), + ("127.0.0.1", self.noauth_port), + } + self.assertGreaterEqual(len(mapped_listener.connection_ids), 4) + for connection_id in mapped_listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + self.assertEqual(connection_id, ("localhost", 12345)) + self.assertGreaterEqual(len(rs_listener.connection_ids), 4) + for connection_id in rs_listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + self.assertIn(connection_id, self.nodes) diff --git a/test/test_socks5.py b/test/test_socks5.py new file mode 100644 index 0000000000..dcd7220cec --- /dev/null +++ b/test/test_socks5.py @@ -0,0 +1,430 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for SOCKS5 proxy connection establishment.""" + +from __future__ import annotations + +import asyncio +import select +import socket +import ssl +import sys +import threading +from types import SimpleNamespace +from typing import Any, Optional, cast +from unittest import mock + +import pytest + +sys.path[0:0] = [""] + +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT +from pymongo.pool_options import PoolOptions, _ProxyOptions +from pymongo.pool_shared import _configured_socket, _create_connection, _Socks5Error +from test import PyMongoTestCase, unittest +from test.helpers_shared import CERT_PATH + +_IS_SYNC = True +pytestmark = pytest.mark.default + + +class _Socks5Server: + def __init__( + self, + method: int = 0, + auth_status: int = 0, + reply: int = 0, + bound_atyp: int = 1, + payload: bytes = b"", + method_version: int = 5, + reply_version: int = 5, + eof_stage: Optional[str] = None, + stall: bool = False, + target: Optional[tuple[str, int]] = None, + ) -> None: + self.method = method + self.auth_status = auth_status + self.reply = reply + self.bound_atyp = bound_atyp + self.payload = payload + self.method_version = method_version + self.reply_version = reply_version + self.eof_stage = eof_stage + self.stall = stall + self.target = target + self.started = threading.Event() + self.greeting = b"" + self.auth = b"" + self.request = b"" + self.error: Optional[BaseException] = None + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.listener.settimeout(3) + self.address = self.listener.getsockname() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + @staticmethod + def _read(sock: socket.socket, size: int) -> bytes: + data = bytearray() + while len(data) < size: + part = sock.recv(size - len(data)) + if not part: + raise EOFError("client closed connection") + data.extend(part) + return bytes(data) + + def _serve(self) -> None: + upstream: Optional[socket.socket] = None + try: + with self.listener.accept()[0] as conn: + conn.settimeout(3) + version, nmethods = self._read(conn, 2) + self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) + self.started.set() + if self.stall: + conn.recv(1) + return + if self.eof_stage == "greeting": + conn.sendall(b"\x05") + return + conn.sendall(bytes((self.method_version, self.method))) + if self.method == 2: + auth_version, username_length = self._read(conn, 2) + username = self._read(conn, username_length) + password_length = self._read(conn, 1)[0] + password = self._read(conn, password_length) + self.auth = ( + bytes((auth_version, username_length)) + + username + + bytes((password_length,)) + + password + ) + if self.eof_stage == "auth": + conn.sendall(b"\x01") + return + conn.sendall(bytes((1, self.auth_status))) + if self.auth_status: + return + version, command, reserved, atyp = self._read(conn, 4) + if atyp == 1: + addr = self._read(conn, 4) + elif atyp == 4: + addr = self._read(conn, 16) + else: + length = self._read(conn, 1) + addr = length + self._read(conn, length[0]) + self.request = ( + bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) + ) + if self.eof_stage == "connect": + conn.sendall(b"\x05\x00") + return + if self.target is not None: + upstream = socket.create_connection(self.target, timeout=3) + if self.bound_atyp == 1: + bound = b"\x01\x7f\x00\x00\x01\x00\x00" + elif self.bound_atyp == 4: + bound = b"\x04" + (b"\x00" * 16) + b"\x00\x00" + else: + bound = b"\x03\x01x\x00\x00" + conn.sendall(bytes((self.reply_version, self.reply, 0)) + bound + self.payload) + if self.payload: + return + if upstream is not None: + conn.setblocking(False) + upstream.setblocking(False) + while True: + readable, _, _ = select.select([conn, upstream], [], [], 5) + if not readable: + return + for source in readable: + data = source.recv(65536) + if not data: + return + destination = upstream if source is conn else conn + destination.sendall(data) + else: + while conn.recv(1024): + pass + except BaseException as exc: + self.error = exc + finally: + if upstream is not None: + upstream.close() + self.listener.close() + + def close(self) -> None: + self.listener.close() + self.thread.join(timeout=3) + + +class TestSocks5(PyMongoTestCase): + def _connect( + self, + server: _Socks5Server, + address: tuple[str, int], + username: Optional[str] = None, + password: Optional[str] = None, + timeout: float = 2, + ) -> socket.socket: + return _create_connection( + address, + PoolOptions( + connect_timeout=timeout, + proxy_options=_ProxyOptions( + server.address[0], server.address[1], username, password + ), + ), + ) + + def test_greeting_and_authenticated_connect(self) -> None: + server = _Socks5Server(method=2) + sock = self._connect(server, ("mongodb.example", 27017), "üser", "påss") + sock.close() + server.close() + self.assertEqual(server.greeting, b"\x05\x02\x00\x02") + self.assertEqual(server.auth, b"\x01\x05" + "üser".encode() + b"\x05" + "påss".encode()) + self.assertEqual(server.request[:4], b"\x05\x01\x00\x03") + self.assertEqual( + server.request[4:], + bytes((len(b"mongodb.example"),)) + b"mongodb.example" + (27017).to_bytes(2, "big"), + ) + + def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: + server = _Socks5Server(method=0) + sock = self._connect(server, ("127.0.0.1", 27017), "user", "password") + sock.close() + server.close() + self.assertEqual(server.greeting, b"\x05\x02\x00\x02") + self.assertEqual(server.auth, b"") + self.assertEqual(server.request, b"\x05\x01\x00\x01\x7f\x00\x00\x01\x69\x89") + + def test_no_credentials_only_offer_no_authentication(self) -> None: + server = _Socks5Server(method=2) + with self.assertRaises(_Socks5Error): + self._connect(server, ("localhost", 27017)) + server.close() + self.assertEqual(server.greeting, b"\x05\x01\x00") + + def test_rejects_all_unacceptable_methods(self) -> None: + for method in (0x01, 0xFF): + server = _Socks5Server(method=method) + with self.subTest(method=method), self.assertRaises(_Socks5Error): + self._connect(server, ("localhost", 27017)) + server.close() + + def test_rejects_authentication_failure(self) -> None: + server = _Socks5Server(method=2, auth_status=1) + with self.assertRaises(_Socks5Error): + self._connect(server, ("localhost", 27017), "user", "password") + server.close() + self.assertEqual(server.auth, b"\x01\x04user\x08password") + + def test_connect_request_address_types(self) -> None: + cases = ( + ("192.0.2.1", b"\x01\xc0\x00\x02\x01"), + ("2001:db8::1", b"\x04" + bytes.fromhex("20010db8000000000000000000000001")), + ( + "münich.example", + b"\x03" + + bytes((len("xn--mnich-kva.example"),)) + + "xn--mnich-kva.example".encode("ascii"), + ), + ) + for host, expected in cases: + server = _Socks5Server() + sock = self._connect(server, (host, 27017)) + sock.close() + server.close() + self.assertEqual(server.request, b"\x05\x01\x00" + expected + b"\x69\x89") + + def test_drains_bound_address_and_preserves_early_payload(self) -> None: + for atyp in (1, 3, 4): + server = _Socks5Server(bound_atyp=atyp, payload=b"mongo") + sock = self._connect(server, ("localhost", 27017)) + if _IS_SYNC: + payload = sock.recv(5) + else: + sock.setblocking(False) + payload = asyncio.get_running_loop().sock_recv(sock, 5) + sock.close() + server.close() + self.assertEqual(payload, b"mongo") + + def test_rejects_socks_reply_codes(self) -> None: + for reply in range(1, 9): + server = _Socks5Server(reply=reply) + with self.subTest(reply=reply), self.assertRaises(_Socks5Error): + self._connect(server, ("localhost", 27017)) + server.close() + + def test_proxy_options_reach_server_and_monitor_pools(self) -> None: + from pymongo.synchronous.mongo_client import MongoClient + from pymongo.synchronous.topology import Topology + + client = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) + try: + topology = Topology(client._topology_settings) + server_pool = topology._create_pool_for_server(("localhost", 27017)) + monitor_pool = topology._create_pool_for_monitor(("localhost", 27017)) + expected = _ProxyOptions("127.0.0.1") + self.assertEqual(server_pool.opts._proxy_options, expected) + self.assertEqual(monitor_pool.opts._proxy_options, expected) + finally: + client.close() + + def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(self) -> None: + from pymongo.synchronous.mongo_client import MongoClient + + client = MongoClient( + "mongodb://localhost/?proxyHost=localhost&proxyPort=1234", + proxyHost="[::1]", + proxyPort=5678, + proxyUsername="", + proxyPassword="", + connect=False, + ) + try: + self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1", 5678)) + finally: + client.close() + + def test_bad_versions_and_unexpected_eof_fail(self) -> None: + cases = ( + (_Socks5Server(method_version=4), ("localhost", 27017), None), + (_Socks5Server(reply_version=4), ("localhost", 27017), None), + (_Socks5Server(eof_stage="greeting"), ("localhost", 27017), None), + ( + _Socks5Server(method=2, eof_stage="auth"), + ("localhost", 27017), + ("user", "password"), + ), + (_Socks5Server(eof_stage="connect"), ("localhost", 27017), None), + ) + for server, address, credentials in cases: + with self.subTest(stage=server.eof_stage, version=server.method_version): + with self.assertRaises(_Socks5Error): + self._connect(server, address, *(credentials or (None, None))) + server.close() + + def test_timeout_covers_the_entire_handshake(self) -> None: + server = _Socks5Server(stall=True) + with self.assertRaises(socket.timeout): + self._connect(server, ("localhost", 27017), timeout=0.1) + server.close() + + @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the synchronous driver") + def test_cancellation_closes_proxy_socket(self) -> None: + server = _Socks5Server(stall=True) + task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) + asyncio.to_thread(server.started.wait, 2) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + task + server.close() + + def test_tls_sni_targets_destination_not_proxy(self) -> None: + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + target_address = listener.getsockname() + tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") + sni_names: list[Optional[str]] = [] + tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + + def serve_tls() -> None: + try: + raw, _ = listener.accept() + with tls_context.wrap_socket(raw, server_side=True) as conn: + conn.recv(1) + except OSError: + pass + finally: + listener.close() + + tls_thread = threading.Thread(target=serve_tls, daemon=True) + tls_thread.start() + proxy = _Socks5Server(target=target_address) + client_context = ssl.create_default_context() + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + sock = _configured_socket( + ("mongodb.example", target_address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, client_context), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), + ), + ) + sock.close() + proxy.close() + tls_thread.join(timeout=3) + self.assertEqual(sni_names, ["mongodb.example"]) + + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + def test_kms_request_uses_client_proxy_options(self) -> None: + import pymongo.synchronous.encryption as encryption_module + from pymongo.synchronous.encryption import _EncryptionIO + from pymongo.synchronous.mongo_client import MongoClient + + class FakeOpts: + _kms_connect_callback = None + + @staticmethod + def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: + return {"aws": object()} + + client = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) + expected = _ProxyOptions("127.0.0.1") + io = _EncryptionIO(client, client.keyvault.datakeys, None, FakeOpts()) # type: ignore[arg-type] + connection = mock.MagicMock() + + def connect(_address, options, _callback, _timeout): + self.assertEqual(options._proxy_options, expected) + return connection + + def sendall(_connection, _message): + pass + + try: + with ( + mock.patch.object(encryption_module, "_connect_kms", new=connect), + mock.patch.object(encryption_module, "sendall", new=sendall), + ): + io.kms_request( + SimpleNamespace( + endpoint="kms.example:443", + message=b"request", + kms_provider="aws", + usleep=0, + bytes_needed=0, + ) + ) + finally: + client.close() + connection.close.assert_called_once_with() + + def test_proxy_settings_are_stored_in_pool_options(self) -> None: + from pymongo.synchronous.mongo_client import MongoClient + + client = MongoClient("mongodb://localhost/?proxyHost=[::1]", connect=False) + try: + self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1")) + finally: + client.close() diff --git a/test/test_socks5_prose.py b/test/test_socks5_prose.py new file mode 100644 index 0000000000..b4d12445b3 --- /dev/null +++ b/test/test_socks5_prose.py @@ -0,0 +1,311 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SOCKS5 connection-string prose tests.""" + +from __future__ import annotations + +import os +import select +import socket +import threading +from typing import Optional +from urllib.parse import quote_plus + +import pytest + +from pymongo import monitoring +from pymongo.hello import HelloCompat +from pymongo.synchronous.mongo_client import MongoClient +from test import IntegrationTest, client_context, unittest +from test.helpers_shared import TLS_OPTIONS, db_pwd, db_user + +_IS_SYNC = True +pytestmark = pytest.mark.default + + +class _CommandListener(monitoring.CommandListener): + def __init__(self) -> None: + self.connection_ids: list[tuple[str, Optional[int]]] = [] + + def started(self, event: monitoring.CommandStartedEvent) -> None: + self.connection_ids.append(event.connection_id) + + def succeeded(self, event: monitoring.CommandSucceededEvent) -> None: + self.connection_ids.append(event.connection_id) + + def failed(self, event: monitoring.CommandFailedEvent) -> None: + self.connection_ids.append(event.connection_id) + + +class Socks5Server: + """Small threaded SOCKS5 forwarder for the driver prose tests.""" + + def __init__( + self, + port: int, + mapping: dict[tuple[str, int], tuple[str, int]], + credentials: Optional[tuple[str, str]] = None, + ) -> None: + self.mapping = mapping + self.credentials = credentials + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + try: + self.listener.bind(("127.0.0.1", port)) + except OSError: + self.listener.bind(("127.0.0.1", 0)) + self.port = self.listener.getsockname()[1] + self.listener.listen() + self.listener.settimeout(0.5) + self._closed = threading.Event() + self._threads: list[threading.Thread] = [] + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + @staticmethod + def _read(sock: socket.socket, length: int) -> bytes: + data = bytearray() + while len(data) < length: + part = sock.recv(length - len(data)) + if not part: + raise EOFError + data.extend(part) + return bytes(data) + + def _serve_client(self, client: socket.socket) -> None: + upstream: Optional[socket.socket] = None + try: + client.settimeout(5) + version, method_count = self._read(client, 2) + methods = self._read(client, method_count) + if version != 5: + return + if self.credentials is not None: + method = 2 if 2 in methods else 0xFF + else: + method = 0 if 0 in methods else 0xFF + client.sendall(bytes((5, method))) + if method == 0xFF: + return + if method == 2: + version, username_length = self._read(client, 2) + username = self._read(client, username_length) + password_length = self._read(client, 1)[0] + password = self._read(client, password_length) + user, secret = self.credentials or ("", "") + status = ( + 0 + if version == 1 and username == user.encode() and password == secret.encode() + else 1 + ) + client.sendall(bytes((1, status))) + if status: + return + version, command, reserved, atyp = self._read(client, 4) + if atyp == 1: + host = socket.inet_ntop(socket.AF_INET, self._read(client, 4)) + elif atyp == 4: + host = socket.inet_ntop(socket.AF_INET6, self._read(client, 16)) + elif atyp == 3: + length = self._read(client, 1)[0] + host = self._read(client, length).decode("idna") + else: + client.sendall(b"\x05\x08\x00\x01\x00\x00\x00\x00\x00\x00") + return + port = int.from_bytes(self._read(client, 2), "big") + if version != 5 or command != 1 or reserved != 0: + client.sendall(b"\x05\x07\x00\x01\x00\x00\x00\x00\x00\x00") + return + target = self.mapping.get((host.lower(), port), (host, port)) + try: + upstream = socket.create_connection(target, timeout=5) + except OSError: + client.sendall(b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00") + return + bound_host, bound_port = upstream.getsockname()[:2] + try: + bound_ip = socket.inet_pton(socket.AF_INET, bound_host) + atyp = 1 + except OSError: + bound_ip = socket.inet_pton(socket.AF_INET6, bound_host) + atyp = 4 + client.sendall(bytes((5, 0, 0, atyp)) + bound_ip + int(bound_port).to_bytes(2, "big")) + client.setblocking(False) + upstream.setblocking(False) + while True: + readable, _, _ = select.select([client, upstream], [], [], 10) + if not readable: + continue + for source in readable: + data = source.recv(65536) + if not data: + return + target_sock = upstream if source is client else client + target_sock.sendall(data) + except (OSError, EOFError): + pass + finally: + if upstream is not None: + upstream.close() + client.close() + + def _serve(self) -> None: + while not self._closed.is_set(): + try: + client, _ = self.listener.accept() + except socket.timeout: + continue + except OSError: + break + thread = threading.Thread(target=self._serve_client, args=(client,), daemon=True) + self._threads.append(thread) + thread.start() + + def close(self) -> None: + self._closed.set() + self.listener.close() + self._thread.join(timeout=2) + for thread in self._threads: + thread.join(timeout=2) + + +class TestSocks5Prose(IntegrationTest): + @client_context.require_replica_set + def setUp(self) -> None: + super().setUp() + self.nodes = sorted(client_context.nodes) + self.mapping = {("localhost", 12345): self.nodes[0]} + self.credentials = getattr(self, "credentials", {}) + if os.environ.get("SOCKS5_PROXIES_STARTED"): + self.auth_port = 1080 + self.noauth_port = 1081 + else: + auth_proxy = Socks5Server(1080, self.mapping, ("username", "p4ssw0rd")) + noauth_proxy = Socks5Server(1081, self.mapping) + self.addCleanup(auth_proxy.close) + self.addCleanup(noauth_proxy.close) + self.auth_port = auth_proxy.port + self.noauth_port = noauth_proxy.port + + def _hosts(self, mapped: bool) -> str: + if mapped: + return "localhost:12345" + return ",".join( + f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" + for host, port in self.nodes + ) + + def _run_case( + self, + hosts: str, + port: int, + credentials: Optional[tuple[str, str]], + succeeds: bool, + use_uri: bool, + listener: Optional[_CommandListener] = None, + ) -> None: + proxy_options = {"proxyHost": "localhost", "proxyPort": port} + if credentials: + proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials + options = dict(TLS_OPTIONS) + options.update(self.credentials) + options.update(proxy_options) + options["serverSelectionTimeoutMS"] = 2500 + if hosts == "localhost:12345": + options["directConnection"] = True + if listener is not None: + options["event_listeners"] = [listener] + if use_uri: + uri_options = "&".join( + f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() + ) + uri_options += "&serverSelectionTimeoutMS=2500" + if hosts == "localhost:12345": + uri_options += "&directConnection=true" + if TLS_OPTIONS: + uri_options += "&" + "&".join( + f"{key}={quote_plus(str(value))}" for key, value in TLS_OPTIONS.items() + ) + auth = "" + if self.credentials: + auth = f"{quote_plus(self.credentials['username'])}:{quote_plus(self.credentials['password'])}@" + client = MongoClient( + f"mongodb://{auth}{hosts}/?{uri_options}", + event_listeners=options.get("event_listeners"), + ) + else: + client = MongoClient(hosts, **options) + try: + if succeeds: + client.admin.command(HelloCompat.CMD) + if listener is not None: + from pymongo.errors import OperationFailure + + with self.assertRaises(OperationFailure): + client.admin.command("nonexistent_socks5_test_command") + else: + from pymongo.errors import ConnectionFailure + + with self.assertRaises(ConnectionFailure): + client.admin.command(HelloCompat.CMD) + finally: + client.close() + + def test_connection_string_prose_table(self) -> None: + cases = ( + (True, self.auth_port, None, False), + (True, self.noauth_port, None, True), + (False, self.auth_port, None, False), + (False, self.noauth_port, None, True), + (True, self.auth_port, ("nonexistentuser", "badauth"), False), + (True, self.noauth_port, ("nonexistentuser", "badauth"), True), + (False, self.noauth_port, ("nonexistentuser", "badauth"), True), + (True, self.auth_port, ("username", "p4ssw0rd"), True), + (True, self.noauth_port, None, True), + (False, self.auth_port, ("username", "p4ssw0rd"), True), + (False, self.noauth_port, None, True), + ) + mapped_listener = _CommandListener() + rs_listener = _CommandListener() + for index, (mapped, proxy_port, credentials, succeeds) in enumerate(cases): + for use_uri in (True, False): + listener = None + if not use_uri: + if index == 1: + listener = mapped_listener + elif index == 3: + listener = rs_listener + self._run_case( + self._hosts(mapped), + proxy_port, + credentials, + succeeds, + use_uri, + listener, + ) + proxy_addresses = { + ("localhost", self.auth_port), + ("localhost", self.noauth_port), + ("127.0.0.1", self.auth_port), + ("127.0.0.1", self.noauth_port), + } + self.assertGreaterEqual(len(mapped_listener.connection_ids), 4) + for connection_id in mapped_listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + self.assertEqual(connection_id, ("localhost", 12345)) + self.assertGreaterEqual(len(rs_listener.connection_ids), 4) + for connection_id in rs_listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + self.assertIn(connection_id, self.nodes) diff --git a/test/test_uri_parser.py b/test/test_uri_parser.py index 9ee9ace759..1a0a3b22d1 100644 --- a/test/test_uri_parser.py +++ b/test/test_uri_parser.py @@ -647,6 +647,63 @@ def test_split_options_duplicate_warning(self): def test_split_options_empty_authsource(self): self.assertRaises(InvalidURI, split_options, "authSource=") + def test_socks5_proxy_options(self): + parsed = parse_uri( + "mongodb://localhost/?proxyHost=%5B%3A%3A1%5D&proxyPort=1234" + "&proxyUsername=user&proxyPassword=pass" + ) + self.assertEqual( + parsed["options"], + { + "proxyHost": "::1", + "proxyPort": 1234, + "proxyUsername": "user", + "proxyPassword": "pass", + }, + ) + empty_credentials = parse_uri( + "mongodb://localhost/?proxyHost=localhost&proxyUsername=&proxyPassword=" + ) + self.assertEqual(empty_credentials["options"], {"proxyHost": "localhost"}) + self.assertEqual( + parse_uri("mongodb://localhost/?proxyHost=localhost")["options"]["proxyHost"], + "localhost", + ) + + def test_socks5_invalid_proxy_options(self): + for query in ( + "proxyHost=", + "proxyHost=%5B%3A%3A1", + "proxyHost=%5B%3A%3A1%5D%3A1080", + "proxyHost=localhost%3A1080", + "proxyHost=local%20host", + "proxyPort=1080", + "proxyUsername=user", + "proxyPassword=pass", + "proxyHost=localhost&proxyUsername=user", + "proxyHost=localhost&proxyPassword=pass", + "proxyHost=localhost&proxyPort=0", + "proxyHost=localhost&proxyPort=-1", + "proxyHost=localhost&proxyPort=65536", + "proxyHost=localhost&proxyPort=abc", + "proxyHost=localhost&proxyUsername=" + ("é" * 128) + "&proxyPassword=pass", + "proxyHost=localhost&proxyUsername=user&proxyPassword=" + ("é" * 128), + ): + with ( + self.subTest(query=query[:60]), + self.assertRaises((InvalidURI, ValueError, ConfigurationError)), + ): + parse_uri("mongodb://localhost/?" + query) + + for option in ("proxyhost", "proxyport", "proxyusername", "proxypassword"): + uri = "mongodb://localhost/?proxyHost=localhost&" + option + "=a&" + option + "=b" + with self.subTest(option=option), self.assertRaises(InvalidURI): + parse_uri(uri) + + def test_socks5_proxy_rejects_unix_socket(self): + with self.assertRaises(ConfigurationError): + parse_uri("mongodb://%2Ftmp%2Fmongodb.sock/?proxyHost=localhost") + def test_check_options_conflicts(self): self.assertRaises( ConfigurationError, diff --git a/test/uri_options/proxy-options.json b/test/uri_options/proxy-options.json new file mode 100644 index 0000000000..585546ead7 --- /dev/null +++ b/test/uri_options/proxy-options.json @@ -0,0 +1,139 @@ +{ + "tests": [ + { + "description": "proxyPort without proxyHost", + "uri": "mongodb://localhost/?proxyPort=1080", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "proxyUsername without proxyHost", + "uri": "mongodb://localhost/?proxyUsername=abc", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "proxyPassword without proxyHost", + "uri": "mongodb://localhost/?proxyPassword=def", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "all other proxy options without proxyHost", + "uri": "mongodb://localhost/?proxyPort=1080&proxyUsername=abc&proxyPassword=def", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "proxyUsername without proxyPassword", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyUsername=abc", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "proxyPassword without proxyUsername", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyPassword=def", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "multiple proxyHost parameters", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyHost=localhost2", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "multiple proxyPort parameters", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyPort=1234&proxyPort=12345", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "multiple proxyUsername parameters", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyUsername=abc&proxyUsername=def&proxyPassword=123", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "multiple proxyPassword parameters", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyUsername=abc&proxyPassword=123&proxyPassword=456", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "only host present", + "uri": "mongodb://localhost/?proxyHost=localhost", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + }, + { + "description": "host and default port present", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyPort=1080", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + }, + { + "description": "host and non-default port present", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyPort=12345", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + }, + { + "description": "replicaset, host and non-default port present", + "uri": "mongodb://rs1,rs2,rs3/?proxyHost=localhost&proxyPort=12345", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + }, + { + "description": "all options present", + "uri": "mongodb://rs1,rs2,rs3/?proxyHost=localhost&proxyPort=12345&proxyUsername=asdf&proxyPassword=qwerty", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + } + ] +} From 37873e1ff91fa81b4c255d7e376a0dd8374e3e32 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 02:02:44 +0000 Subject: [PATCH 02/16] PYTHON-2943 Address code review feedback for SOCKS5 support Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .evergreen/remove-unimplemented-tests.sh | 2 - pymongo/asynchronous/encryption.py | 8 +- pymongo/asynchronous/mongo_client.py | 4 +- pymongo/common.py | 10 +- pymongo/pool_options.py | 8 +- pymongo/pool_shared.py | 199 ++++++++++++----------- pymongo/synchronous/encryption.py | 8 +- pymongo/synchronous/mongo_client.py | 4 +- pymongo/uri_parser_shared.py | 30 ++-- test/asynchronous/test_socks5.py | 107 ++++++++++++ test/asynchronous/test_socks5_prose.py | 4 +- test/test_socks5.py | 107 ++++++++++++ test/test_socks5_prose.py | 4 +- 13 files changed, 367 insertions(+), 128 deletions(-) diff --git a/.evergreen/remove-unimplemented-tests.sh b/.evergreen/remove-unimplemented-tests.sh index 7b2b3b02eb..fa4eda65da 100755 --- a/.evergreen/remove-unimplemented-tests.sh +++ b/.evergreen/remove-unimplemented-tests.sh @@ -46,8 +46,6 @@ find /$PYMONGO/test -type f -name 'pre-42-*.json' -delete rm $PYMONGO/test/csot/override-collection-timeoutMS.json rm $PYMONGO/test/csot/override-database-timeoutMS.json -# PYTHON-2943 - Socks5 Proxy Support - # PYTHON-5517 - Avoid clearing the connection pool when the server connection rate limiter triggers rm $PYMONGO/test/discovery_and_monitoring/unified/backpressure-*.json diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index adfc3e3abd..30800109a9 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -123,6 +123,7 @@ def __init__( key_vault_coll: AsyncCollection[_DocumentTypeArg], mongocryptd_client: Optional[AsyncMongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, + proxy_client: Optional[AsyncMongoClient[_DocumentTypeArg]] = None, ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -140,8 +141,8 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client - proxy_client = client if client is not None else key_vault_coll.database.client - self._proxy_options = proxy_client.options.pool_options._proxy_options + source_client = proxy_client or client or key_vault_coll.database.client + self._proxy_options = source_client.options.pool_options._proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) @@ -412,6 +413,7 @@ def _get_internal_client( key_vault_coll, # type:ignore[arg-type] mongocryptd_client, opts, + proxy_client=client, ) self._auto_encrypter = AsyncAutoEncrypter( io_callbacks, @@ -659,7 +661,7 @@ def __init__( ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( - None, key_vault_coll, None, opts + None, key_vault_coll, None, opts, proxy_client=key_vault_client ) self._encryption = AsyncExplicitEncrypter( self._io_callbacks, diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 66f0c58f16..c57bc9d351 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -152,6 +152,8 @@ Coroutine[Any, Any, T], ] +_REDACTED_REPR_OPTS = frozenset(["username", "password", "proxyusername", "proxypassword"]) + _IS_SYNC = False _WriteOp = Union[ @@ -1436,7 +1438,7 @@ def option_repr(option: str, value: Any) -> str: options.extend( option_repr(key, self._options._options[key]) for key in self._options._options - if key not in set(self._constructor_args) and key != "username" and key != "password" + if key not in set(self._constructor_args) and key not in _REDACTED_REPR_OPTS ) return ", ".join(options) diff --git a/pymongo/common.py b/pymongo/common.py index caa1ad5aad..cdf09e083c 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -151,6 +151,10 @@ # Auth mechanism properties that must raise an error instead of warning if they invalidate. _MECH_PROP_MUST_RAISE = ["CANONICALIZE_HOST_NAME"] +# URI options that must raise an error instead of warning if they invalidate, +# so a malformed proxy option never silently falls back to a direct connection. +_URI_OPTS_MUST_RAISE = frozenset(["proxyhost", "proxyport", "proxyusername", "proxypassword"]) + def partition_node(node: str) -> tuple[str, int]: """Split a host:port string into (host, int(port)) pair.""" @@ -230,8 +234,10 @@ def validate_non_negative_integer(option: str, value: Any) -> int: return val -def validate_proxy_host(option: str, value: Any) -> str: +def validate_proxy_host(option: str, value: Any) -> Optional[str]: """Validate and normalize a SOCKS5 proxy host.""" + if value is None: + return None host = validate_string(option, value) if not host: raise ValueError(f"{option} cannot be an empty string") @@ -933,7 +939,7 @@ def get_setter_key(x: str) -> str: validator = _get_validator(opt, URI_OPTIONS_VALIDATOR_MAP, normed_key=normed_key) validated = validator(opt, value) except (ValueError, TypeError, ConfigurationError) as exc: - if ( + if normed_key in _URI_OPTS_MUST_RAISE or ( normed_key == "authmechanismproperties" and any(p in str(exc) for p in _MECH_PROP_MUST_RAISE) and "is not a supported auth mechanism property" not in str(exc) diff --git a/pymongo/pool_options.py b/pymongo/pool_options.py index e8ab729ea0..c86c6fe526 100644 --- a/pymongo/pool_options.py +++ b/pymongo/pool_options.py @@ -25,8 +25,9 @@ import sys from collections.abc import MutableMapping from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Any, NamedTuple, Optional +from typing import TYPE_CHECKING, Any, Optional import bson from pymongo import __version__ @@ -369,13 +370,14 @@ def _truncate_metadata(metadata: MutableMapping[str, Any]) -> None: "foo".encode("idna") -class _ProxyOptions(NamedTuple): +@dataclass(frozen=True, slots=True) +class _ProxyOptions: """Immutable SOCKS5 proxy connection settings.""" host: str port: int = 1080 username: Optional[str] = None - password: Optional[str] = None + password: Optional[str] = field(default=None, repr=False) class PoolOptions: diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 19f8c23ea8..fe4e1a30e3 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -597,6 +597,18 @@ class PoolState: CLOSED = 3 +_SOCKS5_REPLY_REASONS = { + 0x01: "general SOCKS server failure", + 0x02: "connection not allowed by ruleset", + 0x03: "network unreachable", + 0x04: "host unreachable", + 0x05: "connection refused", + 0x06: "TTL expired", + 0x07: "command not supported", + 0x08: "address type not supported", +} + + class _Socks5Error(OSError): """A SOCKS5 proxy connection or protocol failure.""" @@ -625,13 +637,48 @@ def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: return remaining +def _socks5_greeting(proxy: _ProxyOptions) -> tuple[bytes, tuple[int, ...]]: + if proxy.username is not None and proxy.password is not None: + return b"\x05\x02\x00\x02", (0x00, 0x02) + return b"\x05\x01\x00", (0x00,) + + +def _socks5_check_greeting_reply(data: bytes, allowed: tuple[int, ...]) -> int: + version, method = data + if version != 0x05: + raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") + if method not in allowed: + if method == 0xFF: + raise _Socks5Error( + "SOCKS5 initial handshake failed: no acceptable authentication methods" + ) + raise _Socks5Error( + f"SOCKS5 initial handshake failed: unexpected authentication method {method:#04x}" + ) + return method + + +def _socks5_auth_request(proxy: _ProxyOptions) -> bytes: + assert proxy.username is not None + assert proxy.password is not None + username = proxy.username.encode("utf-8") + password = proxy.password.encode("utf-8") + return bytes((0x01, len(username))) + username + bytes((len(password),)) + password + + +def _socks5_check_auth_reply(data: bytes) -> None: + auth_version, status = data + if auth_version != 0x01 or status != 0x00: + raise _Socks5Error("SOCKS5 authentication failed") + + def _socks5_request(address: _Address) -> bytes: host, port = address try: ip = ipaddress.ip_address(host.split("%", 1)[0]) except ValueError: try: - domain = host.encode("idna") + domain = host.encode("ascii") if host.isascii() else host.encode("idna") except UnicodeError as exc: raise _Socks5Error(f"SOCKS5 connect failed: invalid domain name: {exc}") from exc if not 1 <= len(domain) <= 255: @@ -644,6 +691,20 @@ def _socks5_request(address: _Address) -> bytes: return b"\x05\x01\x00" + encoded_host + port.to_bytes(2, "big") +def _socks5_check_connect_reply(data: bytes) -> int: + version, reply, reserved, atyp = data + if version != 0x05: + raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") + if reserved != 0x00: + raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") + if reply: + reason = _SOCKS5_REPLY_REASONS.get(reply, f"unknown error {reply:#04x}") + raise _Socks5Error(f"SOCKS5 connect failed: {reason}") + if atyp not in (0x01, 0x03, 0x04): + raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") + return atyp + + def _socks5_read_exact(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: data = bytearray() while len(data) < size: @@ -658,62 +719,24 @@ def _socks5_read_exact(sock: socket.socket, size: int, deadline: Optional[float] def _socks5_handshake( sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] ) -> None: - has_credentials = proxy.username is not None and proxy.password is not None - greeting = b"\x05\x02\x00\x02" if has_credentials else b"\x05\x01\x00" + greeting, allowed = _socks5_greeting(proxy) sock.settimeout(_proxy_remaining(deadline)) sock.sendall(greeting) - version, method = _socks5_read_exact(sock, 2, deadline) - if version != 0x05: - raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") - allowed = (0x00, 0x02) if has_credentials else (0x00,) - if method not in allowed: - if method == 0xFF: - raise _Socks5Error( - "SOCKS5 initial handshake failed: no acceptable authentication methods" - ) - raise _Socks5Error( - f"SOCKS5 initial handshake failed: unexpected authentication method {method:#04x}" - ) + method = _socks5_check_greeting_reply(_socks5_read_exact(sock, 2, deadline), allowed) if method == 0x02: - assert proxy.username is not None - assert proxy.password is not None - username = proxy.username.encode("utf-8") - password = proxy.password.encode("utf-8") sock.settimeout(_proxy_remaining(deadline)) - sock.sendall(bytes((0x01, len(username))) + username + bytes((len(password),)) + password) - auth_version, status = _socks5_read_exact(sock, 2, deadline) - if auth_version != 0x01 or status != 0x00: - raise _Socks5Error("SOCKS5 authentication failed") + sock.sendall(_socks5_auth_request(proxy)) + _socks5_check_auth_reply(_socks5_read_exact(sock, 2, deadline)) sock.settimeout(_proxy_remaining(deadline)) sock.sendall(_socks5_request(address)) - version, reply, reserved, atyp = _socks5_read_exact(sock, 4, deadline) - if version != 0x05: - raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") - if reserved != 0x00: - raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") - if reply: - reasons = { - 0x01: "general SOCKS server failure", - 0x02: "connection not allowed by ruleset", - 0x03: "network unreachable", - 0x04: "host unreachable", - 0x05: "connection refused", - 0x06: "TTL expired", - 0x07: "command not supported", - 0x08: "address type not supported", - } - reason = reasons.get(reply, f"unknown error {reply:#04x}") - raise _Socks5Error(f"SOCKS5 connect failed: {reason}") + atyp = _socks5_check_connect_reply(_socks5_read_exact(sock, 4, deadline)) if atyp == 0x01: bound_size = 6 elif atyp == 0x04: bound_size = 18 - elif atyp == 0x03: - domain_length = _socks5_read_exact(sock, 1, deadline)[0] - bound_size = domain_length + 2 else: - raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") + bound_size = _socks5_read_exact(sock, 1, deadline)[0] + 2 _socks5_read_exact(sock, bound_size, deadline) @@ -736,61 +759,27 @@ async def _async_socks5_handshake( sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] ) -> None: loop = asyncio.get_running_loop() - has_credentials = proxy.username is not None and proxy.password is not None - greeting = b"\x05\x02\x00\x02" if has_credentials else b"\x05\x01\x00" + greeting, allowed = _socks5_greeting(proxy) await asyncio.wait_for(loop.sock_sendall(sock, greeting), timeout=_proxy_remaining(deadline)) - version, method = await _async_socks5_read_exact(sock, 2, deadline) - if version != 0x05: - raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") - allowed = (0x00, 0x02) if has_credentials else (0x00,) - if method not in allowed: - if method == 0xFF: - raise _Socks5Error( - "SOCKS5 initial handshake failed: no acceptable authentication methods" - ) - raise _Socks5Error( - f"SOCKS5 initial handshake failed: unexpected authentication method {method:#04x}" - ) + method = _socks5_check_greeting_reply( + await _async_socks5_read_exact(sock, 2, deadline), allowed + ) if method == 0x02: - assert proxy.username is not None - assert proxy.password is not None - username = proxy.username.encode("utf-8") - password = proxy.password.encode("utf-8") - auth = bytes((0x01, len(username))) + username + bytes((len(password),)) + password - await asyncio.wait_for(loop.sock_sendall(sock, auth), timeout=_proxy_remaining(deadline)) - auth_version, status = await _async_socks5_read_exact(sock, 2, deadline) - if auth_version != 0x01 or status != 0x00: - raise _Socks5Error("SOCKS5 authentication failed") + await asyncio.wait_for( + loop.sock_sendall(sock, _socks5_auth_request(proxy)), + timeout=_proxy_remaining(deadline), + ) + _socks5_check_auth_reply(await _async_socks5_read_exact(sock, 2, deadline)) await asyncio.wait_for( loop.sock_sendall(sock, _socks5_request(address)), timeout=_proxy_remaining(deadline) ) - version, reply, reserved, atyp = await _async_socks5_read_exact(sock, 4, deadline) - if version != 0x05: - raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") - if reserved != 0x00: - raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") - if reply: - reasons = { - 0x01: "general SOCKS server failure", - 0x02: "connection not allowed by ruleset", - 0x03: "network unreachable", - 0x04: "host unreachable", - 0x05: "connection refused", - 0x06: "TTL expired", - 0x07: "command not supported", - 0x08: "address type not supported", - } - reason = reasons.get(reply, f"unknown error {reply:#04x}") - raise _Socks5Error(f"SOCKS5 connect failed: {reason}") + atyp = _socks5_check_connect_reply(await _async_socks5_read_exact(sock, 4, deadline)) if atyp == 0x01: bound_size = 6 elif atyp == 0x04: bound_size = 18 - elif atyp == 0x03: - domain_length = (await _async_socks5_read_exact(sock, 1, deadline))[0] - bound_size = domain_length + 2 else: - raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") + bound_size = (await _async_socks5_read_exact(sock, 1, deadline))[0] + 2 await _async_socks5_read_exact(sock, bound_size, deadline) @@ -802,9 +791,10 @@ async def _async_create_proxy_connection( if socket.has_ipv6 and proxy.host != "localhost": family = socket.AF_UNSPEC try: + remaining = _proxy_remaining(deadline) getaddrinfo = _getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) - results = await asyncio.wait_for(getaddrinfo, timeout=_proxy_remaining(deadline)) - except asyncio.TimeoutError as exc: + results = await asyncio.wait_for(getaddrinfo, timeout=remaining) + except (asyncio.TimeoutError, socket.timeout) as exc: raise _Socks5Timeout("timed out") from exc except OSError as exc: raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc @@ -828,11 +818,15 @@ async def _async_create_proxy_connection( ) connected = True await _async_socks5_handshake(sock, address, proxy, deadline) - sock.settimeout(_proxy_remaining(deadline)) + post_timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout + sock.settimeout(post_timeout) return sock except asyncio.TimeoutError as exc: sock.close() - raise _Socks5Timeout("timed out") from exc + if connected or (deadline is not None and deadline - time.monotonic() <= 0): + raise _Socks5Timeout("timed out") from exc + err = _Socks5Timeout("timed out") + err.__cause__ = exc except asyncio.CancelledError: sock.close() raise @@ -841,7 +835,11 @@ async def _async_create_proxy_connection( if isinstance(exc, _Socks5Error): raise if isinstance(exc, socket.timeout): - raise _Socks5Timeout("timed out") from exc + if connected or (deadline is not None and deadline - time.monotonic() <= 0): + raise _Socks5Timeout("timed out") from exc + err = _Socks5Timeout("timed out") + err.__cause__ = exc + continue err = exc # A connected proxy's handshake failure must not try a different # resolved proxy address; the failure is specific to the endpoint. @@ -851,6 +849,8 @@ async def _async_create_proxy_connection( sock.close() raise + if isinstance(err, socket.timeout): + raise err if err is not None: raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") @@ -884,10 +884,15 @@ def _create_proxy_connection( sock.connect(sockaddr) connected = True _socks5_handshake(sock, address, proxy, deadline) + post_timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout + sock.settimeout(post_timeout) return sock except socket.timeout as exc: sock.close() - raise _Socks5Timeout("timed out") from exc + if connected or (deadline is not None and deadline - time.monotonic() <= 0): + raise _Socks5Timeout("timed out") from exc + err = _Socks5Timeout("timed out") + err.__cause__ = exc except OSError as exc: sock.close() if isinstance(exc, _Socks5Error): @@ -900,6 +905,8 @@ def _create_proxy_connection( sock.close() raise + if isinstance(err, socket.timeout): + raise err if err is not None: raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 827d69e507..caf83f0fec 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -122,6 +122,7 @@ def __init__( key_vault_coll: Collection[_DocumentTypeArg], mongocryptd_client: Optional[MongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, + proxy_client: Optional[MongoClient[_DocumentTypeArg]] = None, ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -139,8 +140,8 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client - proxy_client = client if client is not None else key_vault_coll.database.client - self._proxy_options = proxy_client.options.pool_options._proxy_options + source_client = proxy_client or client or key_vault_coll.database.client + self._proxy_options = source_client.options.pool_options._proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) @@ -409,6 +410,7 @@ def _get_internal_client( key_vault_coll, # type:ignore[arg-type] mongocryptd_client, opts, + proxy_client=client, ) self._auto_encrypter = AutoEncrypter( io_callbacks, @@ -652,7 +654,7 @@ def __init__( ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( - None, key_vault_coll, None, opts + None, key_vault_coll, None, opts, proxy_client=key_vault_client ) self._encryption = ExplicitEncrypter( self._io_callbacks, diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 981350db59..dab678e045 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -150,6 +150,8 @@ T, ] +_REDACTED_REPR_OPTS = frozenset(["username", "password", "proxyusername", "proxypassword"]) + _IS_SYNC = True _WriteOp = Union[ @@ -1437,7 +1439,7 @@ def option_repr(option: str, value: Any) -> str: options.extend( option_repr(key, self._options._options[key]) for key in self._options._options - if key not in set(self._constructor_args) and key != "username" and key != "password" + if key not in set(self._constructor_args) and key not in _REDACTED_REPR_OPTS ) return ", ".join(options) diff --git a/pymongo/uri_parser_shared.py b/pymongo/uri_parser_shared.py index b85ab8d5d7..3d3f6a366c 100644 --- a/pymongo/uri_parser_shared.py +++ b/pymongo/uri_parser_shared.py @@ -486,19 +486,7 @@ def split_hosts(hosts: str, default_port: Optional[int] = DEFAULT_PORT) -> list[ ) -def _check_options(nodes: Sized, options: Mapping[str, Any]) -> None: - # Ensure directConnection was not True if there are multiple seeds. - if len(nodes) > 1 and options.get("directconnection"): - raise ConfigurationError("Cannot specify multiple hosts with directConnection=true") - - if options.get("loadbalanced"): - if len(nodes) > 1: - raise ConfigurationError("Cannot specify multiple hosts with loadBalanced=true") - if options.get("directconnection"): - raise ConfigurationError("Cannot specify directConnection=true with loadBalanced=true") - if options.get("replicaset"): - raise ConfigurationError("Cannot specify replicaSet with loadBalanced=true") - +def _check_proxy_options(nodes: Sized, options: Mapping[str, Any]) -> None: proxy_host = options.get("proxyhost") proxy_port = options.get("proxyport") proxy_username = options.get("proxyusername") @@ -526,6 +514,22 @@ def _check_options(nodes: Sized, options: Mapping[str, Any]) -> None: raise ConfigurationError("Cannot specify proxyHost with Unix domain sockets") +def _check_options(nodes: Sized, options: Mapping[str, Any]) -> None: + # Ensure directConnection was not True if there are multiple seeds. + if len(nodes) > 1 and options.get("directconnection"): + raise ConfigurationError("Cannot specify multiple hosts with directConnection=true") + + if options.get("loadbalanced"): + if len(nodes) > 1: + raise ConfigurationError("Cannot specify multiple hosts with loadBalanced=true") + if options.get("directconnection"): + raise ConfigurationError("Cannot specify directConnection=true with loadBalanced=true") + if options.get("replicaset"): + raise ConfigurationError("Cannot specify replicaSet with loadBalanced=true") + + _check_proxy_options(nodes, options) + + def _parse_kms_tls_options( kms_tls_options: Optional[Mapping[str, Any]], is_sync: bool, diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index 2c86d10e8f..7fec1a1883 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -428,3 +428,110 @@ async def test_proxy_settings_are_stored_in_pool_options(self) -> None: self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1")) finally: await client.close() + + async def test_proxy_password_is_redacted_in_repr(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + client = AsyncMongoClient( + "mongodb://user:dbsecret@localhost/?proxyHost=127.0.0.1" + "&proxyUsername=proxyuser&proxyPassword=hunter2", + connect=False, + ) + try: + client_repr = repr(client) + proxy_repr = repr(client.options.pool_options._proxy_options) + self.assertNotIn("hunter2", client_repr) + self.assertNotIn("proxyuser", client_repr) + self.assertNotIn("dbsecret", client_repr) + self.assertNotIn("hunter2", proxy_repr) + self.assertIn("127.0.0.1", proxy_repr) + finally: + await client.close() + + async def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + for uri in ( + "mongodb://localhost/?proxyHost=proxy%3A1080", + "mongodb://localhost/?proxyHost=local%20host", + "mongodb://localhost/?proxyHost=localhost&proxyPort=0", + "mongodb://localhost/?proxyHost=localhost&proxyPort=abc", + ): + with self.subTest(uri=uri), self.assertRaises(ValueError): + AsyncMongoClient(uri, connect=False) + + async def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: + from pymongo.asynchronous.pool import Pool + from pymongo.errors import AutoReconnect, NetworkTimeout + from pymongo.pool_shared import _raise_connection_failure, _Socks5Timeout + + pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) + for cause in (_Socks5Error("boom"), _Socks5Timeout("timed out")): + try: + _raise_connection_failure(("localhost", 27017), cause) + except (AutoReconnect, NetworkTimeout) as wrapped: + pool._handle_connection_error(wrapped) + self.assertFalse(wrapped.has_error_label("SystemOverloadedError")) + self.assertFalse(wrapped.has_error_label("RetryableError")) + + async def test_multi_address_proxy_fallback_on_connect_error(self) -> None: + server = _Socks5Server() + self.addCleanup(server.close) + dead_sock = socket.socket() + dead_sock.bind(("127.0.0.1", 0)) + dead_addr = dead_sock.getsockname() + dead_sock.close() + + fake_results = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), + ] + target = ( + "pymongo.pool_shared.socket.getaddrinfo" + if _IS_SYNC + else "pymongo.pool_shared._getaddrinfo" + ) + + async def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: + return fake_results + + replacement: Any = (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo + with mock.patch(target, new=replacement): + sock = await self._connect(server, ("localhost", 27017)) + sock.close() + + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + async def test_kms_proxy_uses_parent_client_even_with_bypass_auto_encryption(self) -> None: + from pymongo.asynchronous.encryption import _EncryptionIO + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + class FakeOpts: + _kms_connect_callback = None + + @staticmethod + def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: + return {"aws": object()} + + parent_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.1", connect=False) + kv_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + try: + io_bypass = _EncryptionIO( + None, + kv_client.keyvault.datakeys, + None, + FakeOpts(), # type: ignore[arg-type] + proxy_client=parent_client, + ) + self.assertEqual(io_bypass._proxy_options, _ProxyOptions("10.0.0.1")) + + io_explicit = _EncryptionIO( + None, + kv_client.keyvault.datakeys, + None, + FakeOpts(), # type: ignore[arg-type] + proxy_client=kv_client, + ) + self.assertEqual(io_explicit._proxy_options, _ProxyOptions("10.0.0.2")) + finally: + await parent_client.close() + await kv_client.close() diff --git a/test/asynchronous/test_socks5_prose.py b/test/asynchronous/test_socks5_prose.py index 605bb1f16a..d3246b085e 100644 --- a/test/asynchronous/test_socks5_prose.py +++ b/test/asynchronous/test_socks5_prose.py @@ -28,8 +28,8 @@ from pymongo import monitoring from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.hello import HelloCompat -from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest -from test.helpers_shared import TLS_OPTIONS, db_pwd, db_user +from test.asynchronous import AsyncIntegrationTest, async_client_context +from test.helpers_shared import TLS_OPTIONS _IS_SYNC = False pytestmark = pytest.mark.default_async diff --git a/test/test_socks5.py b/test/test_socks5.py index dcd7220cec..afdedf613a 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -428,3 +428,110 @@ def test_proxy_settings_are_stored_in_pool_options(self) -> None: self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1")) finally: client.close() + + def test_proxy_password_is_redacted_in_repr(self) -> None: + from pymongo.synchronous.mongo_client import MongoClient + + client = MongoClient( + "mongodb://user:dbsecret@localhost/?proxyHost=127.0.0.1" + "&proxyUsername=proxyuser&proxyPassword=hunter2", + connect=False, + ) + try: + client_repr = repr(client) + proxy_repr = repr(client.options.pool_options._proxy_options) + self.assertNotIn("hunter2", client_repr) + self.assertNotIn("proxyuser", client_repr) + self.assertNotIn("dbsecret", client_repr) + self.assertNotIn("hunter2", proxy_repr) + self.assertIn("127.0.0.1", proxy_repr) + finally: + client.close() + + def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> None: + from pymongo.synchronous.mongo_client import MongoClient + + for uri in ( + "mongodb://localhost/?proxyHost=proxy%3A1080", + "mongodb://localhost/?proxyHost=local%20host", + "mongodb://localhost/?proxyHost=localhost&proxyPort=0", + "mongodb://localhost/?proxyHost=localhost&proxyPort=abc", + ): + with self.subTest(uri=uri), self.assertRaises(ValueError): + MongoClient(uri, connect=False) + + def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: + from pymongo.errors import AutoReconnect, NetworkTimeout + from pymongo.pool_shared import _raise_connection_failure, _Socks5Timeout + from pymongo.synchronous.pool import Pool + + pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) + for cause in (_Socks5Error("boom"), _Socks5Timeout("timed out")): + try: + _raise_connection_failure(("localhost", 27017), cause) + except (AutoReconnect, NetworkTimeout) as wrapped: + pool._handle_connection_error(wrapped) + self.assertFalse(wrapped.has_error_label("SystemOverloadedError")) + self.assertFalse(wrapped.has_error_label("RetryableError")) + + def test_multi_address_proxy_fallback_on_connect_error(self) -> None: + server = _Socks5Server() + self.addCleanup(server.close) + dead_sock = socket.socket() + dead_sock.bind(("127.0.0.1", 0)) + dead_addr = dead_sock.getsockname() + dead_sock.close() + + fake_results = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), + ] + target = ( + "pymongo.pool_shared.socket.getaddrinfo" + if _IS_SYNC + else "pymongo.pool_shared._getaddrinfo" + ) + + def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: + return fake_results + + replacement: Any = (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo + with mock.patch(target, new=replacement): + sock = self._connect(server, ("localhost", 27017)) + sock.close() + + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + def test_kms_proxy_uses_parent_client_even_with_bypass_auto_encryption(self) -> None: + from pymongo.synchronous.encryption import _EncryptionIO + from pymongo.synchronous.mongo_client import MongoClient + + class FakeOpts: + _kms_connect_callback = None + + @staticmethod + def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: + return {"aws": object()} + + parent_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.1", connect=False) + kv_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + try: + io_bypass = _EncryptionIO( + None, + kv_client.keyvault.datakeys, + None, + FakeOpts(), # type: ignore[arg-type] + proxy_client=parent_client, + ) + self.assertEqual(io_bypass._proxy_options, _ProxyOptions("10.0.0.1")) + + io_explicit = _EncryptionIO( + None, + kv_client.keyvault.datakeys, + None, + FakeOpts(), # type: ignore[arg-type] + proxy_client=kv_client, + ) + self.assertEqual(io_explicit._proxy_options, _ProxyOptions("10.0.0.2")) + finally: + parent_client.close() + kv_client.close() diff --git a/test/test_socks5_prose.py b/test/test_socks5_prose.py index b4d12445b3..52a2d65484 100644 --- a/test/test_socks5_prose.py +++ b/test/test_socks5_prose.py @@ -28,8 +28,8 @@ from pymongo import monitoring from pymongo.hello import HelloCompat from pymongo.synchronous.mongo_client import MongoClient -from test import IntegrationTest, client_context, unittest -from test.helpers_shared import TLS_OPTIONS, db_pwd, db_user +from test import IntegrationTest, client_context +from test.helpers_shared import TLS_OPTIONS _IS_SYNC = True pytestmark = pytest.mark.default From 2df677296cec566ee38cd4e9338a8a3c6e61402b Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 02:10:41 +0000 Subject: [PATCH 03/16] PYTHON-2943 Address Copilot PR review comments Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pymongo/asynchronous/mongo_client.py | 2 + pymongo/asynchronous/uri_parser.py | 3 +- pymongo/synchronous/mongo_client.py | 2 + pymongo/synchronous/uri_parser.py | 3 +- pymongo/uri_parser_shared.py | 8 ++-- test/asynchronous/test_socks5.py | 60 +++++++++++++++++++++----- test/asynchronous/test_socks5_prose.py | 7 +-- test/test_socks5.py | 60 +++++++++++++++++++++----- test/test_socks5_prose.py | 7 +-- 9 files changed, 119 insertions(+), 33 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index c57bc9d351..959697e51b 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -881,6 +881,7 @@ def validator(host: str) -> bool: warn=True, normalize=False, srv_max_hosts=srv_max_hosts, + validate_proxy=False, ) is_srv = entity.startswith(SRV_SCHEME) self._seeds.update(res["nodelist"]) @@ -1006,6 +1007,7 @@ async def _resolve_srv(self) -> None: srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, srv_host_validator=srv_host_validator, + validate_proxy=False, ) seeds.update(res["nodelist"]) opts = res["options"] diff --git a/pymongo/asynchronous/uri_parser.py b/pymongo/asynchronous/uri_parser.py index c42add6ae8..821cf534ec 100644 --- a/pymongo/asynchronous/uri_parser.py +++ b/pymongo/asynchronous/uri_parser.py @@ -136,6 +136,7 @@ async def _parse_srv( srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, srv_host_validator: Optional[Callable[[str], bool]] = None, + validate_proxy: bool = True, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -198,7 +199,7 @@ async def _parse_srv( else: nodes = split_hosts(hosts, default_port=default_port) - _check_options(nodes, options) + _check_options(nodes, options, validate_proxy=validate_proxy) return { "nodelist": nodes, diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index dab678e045..7988b5a8e7 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -882,6 +882,7 @@ def validator(host: str) -> bool: warn=True, normalize=False, srv_max_hosts=srv_max_hosts, + validate_proxy=False, ) is_srv = entity.startswith(SRV_SCHEME) self._seeds.update(res["nodelist"]) @@ -1007,6 +1008,7 @@ def _resolve_srv(self) -> None: srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, srv_host_validator=srv_host_validator, + validate_proxy=False, ) seeds.update(res["nodelist"]) opts = res["options"] diff --git a/pymongo/synchronous/uri_parser.py b/pymongo/synchronous/uri_parser.py index 92608b5efe..fe9331f5c2 100644 --- a/pymongo/synchronous/uri_parser.py +++ b/pymongo/synchronous/uri_parser.py @@ -136,6 +136,7 @@ def _parse_srv( srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, srv_host_validator: Optional[Callable[[str], bool]] = None, + validate_proxy: bool = True, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -198,7 +199,7 @@ def _parse_srv( else: nodes = split_hosts(hosts, default_port=default_port) - _check_options(nodes, options) + _check_options(nodes, options, validate_proxy=validate_proxy) return { "nodelist": nodes, diff --git a/pymongo/uri_parser_shared.py b/pymongo/uri_parser_shared.py index 3d3f6a366c..45c99f1110 100644 --- a/pymongo/uri_parser_shared.py +++ b/pymongo/uri_parser_shared.py @@ -514,7 +514,7 @@ def _check_proxy_options(nodes: Sized, options: Mapping[str, Any]) -> None: raise ConfigurationError("Cannot specify proxyHost with Unix domain sockets") -def _check_options(nodes: Sized, options: Mapping[str, Any]) -> None: +def _check_options(nodes: Sized, options: Mapping[str, Any], validate_proxy: bool = True) -> None: # Ensure directConnection was not True if there are multiple seeds. if len(nodes) > 1 and options.get("directconnection"): raise ConfigurationError("Cannot specify multiple hosts with directConnection=true") @@ -527,7 +527,8 @@ def _check_options(nodes: Sized, options: Mapping[str, Any]) -> None: if options.get("replicaset"): raise ConfigurationError("Cannot specify replicaSet with loadBalanced=true") - _check_proxy_options(nodes, options) + if validate_proxy: + _check_proxy_options(nodes, options) def _parse_kms_tls_options( @@ -573,6 +574,7 @@ def _validate_uri( warn: bool = False, normalize: bool = True, srv_max_hosts: Optional[int] = None, + validate_proxy: bool = True, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -652,7 +654,7 @@ def _validate_uri( else: nodes = split_hosts(hosts, default_port=default_port) - _check_options(nodes, options) + _check_options(nodes, options, validate_proxy=validate_proxy) return { "nodelist": nodes, diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index 7fec1a1883..920449663d 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -432,26 +432,64 @@ async def test_proxy_settings_are_stored_in_pool_options(self) -> None: async def test_proxy_password_is_redacted_in_repr(self) -> None: from pymongo.asynchronous.mongo_client import AsyncMongoClient - client = AsyncMongoClient( - "mongodb://user:dbsecret@localhost/?proxyHost=127.0.0.1" - "&proxyUsername=proxyuser&proxyPassword=hunter2", + for client in ( + AsyncMongoClient( + "mongodb://user:dbsecret@localhost/?proxyHost=127.0.0.1" + "&proxyUsername=proxyuser&proxyPassword=hunter2", + connect=False, + ), + AsyncMongoClient( + "mongodb://localhost/", + username="user", + password="dbsecret", + proxyHost="127.0.0.1", + proxyUsername="proxyuser", + proxyPassword="hunter2", + connect=False, + ), + ): + try: + client_repr = repr(client) + proxy_repr = repr(client.options.pool_options._proxy_options) + self.assertNotIn("hunter2", client_repr) + self.assertNotIn("proxyuser", client_repr) + self.assertNotIn("dbsecret", client_repr) + self.assertNotIn("hunter2", proxy_repr) + self.assertIn("127.0.0.1", proxy_repr) + finally: + await client.close() + + async def test_split_proxy_options_across_uri_and_kwargs(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + client1 = AsyncMongoClient( + "mongodb://localhost/?proxyPort=1081", + proxyHost="proxy.example", + connect=False, + ) + client2 = AsyncMongoClient( + "mongodb://localhost/?proxyHost=proxy.example&proxyUsername=u", + proxyPassword="p", connect=False, ) try: - client_repr = repr(client) - proxy_repr = repr(client.options.pool_options._proxy_options) - self.assertNotIn("hunter2", client_repr) - self.assertNotIn("proxyuser", client_repr) - self.assertNotIn("dbsecret", client_repr) - self.assertNotIn("hunter2", proxy_repr) - self.assertIn("127.0.0.1", proxy_repr) + self.assertEqual( + client1.options.pool_options._proxy_options, + _ProxyOptions("proxy.example", 1081), + ) + self.assertEqual( + client2.options.pool_options._proxy_options, + _ProxyOptions("proxy.example", 1080, "u", "p"), + ) finally: - await client.close() + await client1.close() + await client2.close() async def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> None: from pymongo.asynchronous.mongo_client import AsyncMongoClient for uri in ( + "mongodb://localhost/?proxyHost=", "mongodb://localhost/?proxyHost=proxy%3A1080", "mongodb://localhost/?proxyHost=local%20host", "mongodb://localhost/?proxyHost=localhost&proxyPort=0", diff --git a/test/asynchronous/test_socks5_prose.py b/test/asynchronous/test_socks5_prose.py index d3246b085e..dca5e2f5c0 100644 --- a/test/asynchronous/test_socks5_prose.py +++ b/test/asynchronous/test_socks5_prose.py @@ -219,7 +219,8 @@ async def _run_case( proxy_options = {"proxyHost": "localhost", "proxyPort": port} if credentials: proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials - options = dict(TLS_OPTIONS) + tls_options = dict(TLS_OPTIONS) if async_client_context.tls else {} + options = dict(tls_options) options.update(self.credentials) options.update(proxy_options) options["serverSelectionTimeoutMS"] = 2500 @@ -234,9 +235,9 @@ async def _run_case( uri_options += "&serverSelectionTimeoutMS=2500" if hosts == "localhost:12345": uri_options += "&directConnection=true" - if TLS_OPTIONS: + if tls_options: uri_options += "&" + "&".join( - f"{key}={quote_plus(str(value))}" for key, value in TLS_OPTIONS.items() + f"{key}={quote_plus(str(value))}" for key, value in tls_options.items() ) auth = "" if self.credentials: diff --git a/test/test_socks5.py b/test/test_socks5.py index afdedf613a..ee3c1118de 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -432,26 +432,64 @@ def test_proxy_settings_are_stored_in_pool_options(self) -> None: def test_proxy_password_is_redacted_in_repr(self) -> None: from pymongo.synchronous.mongo_client import MongoClient - client = MongoClient( - "mongodb://user:dbsecret@localhost/?proxyHost=127.0.0.1" - "&proxyUsername=proxyuser&proxyPassword=hunter2", + for client in ( + MongoClient( + "mongodb://user:dbsecret@localhost/?proxyHost=127.0.0.1" + "&proxyUsername=proxyuser&proxyPassword=hunter2", + connect=False, + ), + MongoClient( + "mongodb://localhost/", + username="user", + password="dbsecret", + proxyHost="127.0.0.1", + proxyUsername="proxyuser", + proxyPassword="hunter2", + connect=False, + ), + ): + try: + client_repr = repr(client) + proxy_repr = repr(client.options.pool_options._proxy_options) + self.assertNotIn("hunter2", client_repr) + self.assertNotIn("proxyuser", client_repr) + self.assertNotIn("dbsecret", client_repr) + self.assertNotIn("hunter2", proxy_repr) + self.assertIn("127.0.0.1", proxy_repr) + finally: + client.close() + + def test_split_proxy_options_across_uri_and_kwargs(self) -> None: + from pymongo.synchronous.mongo_client import MongoClient + + client1 = MongoClient( + "mongodb://localhost/?proxyPort=1081", + proxyHost="proxy.example", + connect=False, + ) + client2 = MongoClient( + "mongodb://localhost/?proxyHost=proxy.example&proxyUsername=u", + proxyPassword="p", connect=False, ) try: - client_repr = repr(client) - proxy_repr = repr(client.options.pool_options._proxy_options) - self.assertNotIn("hunter2", client_repr) - self.assertNotIn("proxyuser", client_repr) - self.assertNotIn("dbsecret", client_repr) - self.assertNotIn("hunter2", proxy_repr) - self.assertIn("127.0.0.1", proxy_repr) + self.assertEqual( + client1.options.pool_options._proxy_options, + _ProxyOptions("proxy.example", 1081), + ) + self.assertEqual( + client2.options.pool_options._proxy_options, + _ProxyOptions("proxy.example", 1080, "u", "p"), + ) finally: - client.close() + client1.close() + client2.close() def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> None: from pymongo.synchronous.mongo_client import MongoClient for uri in ( + "mongodb://localhost/?proxyHost=", "mongodb://localhost/?proxyHost=proxy%3A1080", "mongodb://localhost/?proxyHost=local%20host", "mongodb://localhost/?proxyHost=localhost&proxyPort=0", diff --git a/test/test_socks5_prose.py b/test/test_socks5_prose.py index 52a2d65484..3c0a28ea44 100644 --- a/test/test_socks5_prose.py +++ b/test/test_socks5_prose.py @@ -219,7 +219,8 @@ def _run_case( proxy_options = {"proxyHost": "localhost", "proxyPort": port} if credentials: proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials - options = dict(TLS_OPTIONS) + tls_options = dict(TLS_OPTIONS) if client_context.tls else {} + options = dict(tls_options) options.update(self.credentials) options.update(proxy_options) options["serverSelectionTimeoutMS"] = 2500 @@ -234,9 +235,9 @@ def _run_case( uri_options += "&serverSelectionTimeoutMS=2500" if hosts == "localhost:12345": uri_options += "&directConnection=true" - if TLS_OPTIONS: + if tls_options: uri_options += "&" + "&".join( - f"{key}={quote_plus(str(value))}" for key, value in TLS_OPTIONS.items() + f"{key}={quote_plus(str(value))}" for key, value in tls_options.items() ) auth = "" if self.credentials: From bd6b7b48d85e21157842d810b3d9b6be4e2e2694 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 02:22:06 +0000 Subject: [PATCH 04/16] PYTHON-2943 Improve SOCKS5 unit test diff coverage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/asynchronous/test_socks5.py | 167 +++++++++++++++++++++++++++++++ test/test_socks5.py | 167 +++++++++++++++++++++++++++++++ 2 files changed, 334 insertions(+) diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index 920449663d..fca790e69d 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -573,3 +573,170 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: finally: await parent_client.close() await kv_client.close() + + async def test_socks5_helper_edge_cases(self) -> None: + from pymongo.common import validate_proxy_host, validate_proxy_port + from pymongo.pool_shared import ( + _proxy_deadline, + _proxy_remaining, + _socks5_check_connect_reply, + _socks5_request, + _Socks5Timeout, + ) + + self.assertIsNone(validate_proxy_host("proxyHost", None)) + self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") + with self.assertRaises(ValueError): + validate_proxy_host("proxyHost", "[not-ipv6]") + with self.assertRaises(ValueError): + validate_proxy_host("proxyHost", "bad/host") + self.assertIsNone(validate_proxy_port("proxyPort", None)) + with self.assertRaises(TypeError): + validate_proxy_port("proxyPort", True) + + self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) + self.assertIsNone(_proxy_remaining(None)) + with self.assertRaises(_Socks5Timeout): + _proxy_remaining(0.0) + with ( + mock.patch("pymongo.pool_shared._csot.remaining", return_value=0.0), + self.assertRaises(_Socks5Timeout), + ): + _proxy_deadline(PoolOptions(connect_timeout=1)) + + with self.assertRaisesRegex(_Socks5Error, "invalid domain name length"): + _socks5_request(("a" * 256, 27017)) + with self.assertRaisesRegex(_Socks5Error, "invalid domain name"): + _socks5_request((".münich.example", 27017)) + with self.assertRaisesRegex(_Socks5Error, "destination port is required"): + _socks5_request(("localhost", None)) # type: ignore[arg-type] + with self.assertRaisesRegex(_Socks5Error, "invalid reserved byte"): + _socks5_check_connect_reply(b"\x05\x00\x01\x01") + with self.assertRaisesRegex(_Socks5Error, "invalid address type"): + _socks5_check_connect_reply(b"\x05\x00\x00\x09") + + async def test_proxy_connection_error_paths(self) -> None: + proxy_opts = PoolOptions( + connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080) + ) + target = ( + "pymongo.pool_shared.socket.getaddrinfo" + if _IS_SYNC + else "pymongo.pool_shared._getaddrinfo" + ) + + # 1. getaddrinfo raises OSError + def raise_gai(*_a: Any, **_kw: Any) -> Any: + raise socket.gaierror("resolution failed") + + with ( + mock.patch(target, side_effect=raise_gai), + self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), + ): + await _async_create_connection(("localhost", 27017), proxy_opts) + + # 2. getaddrinfo returns empty list + async def empty_async(*_a: Any, **_kw: Any) -> Any: + return [] + + replacement_empty: Any = (lambda *_a, **_kw: []) if _IS_SYNC else empty_async + with ( + mock.patch(target, new=replacement_empty), + self.assertRaisesRegex(_Socks5Error, "returned no addresses"), + ): + await _async_create_connection(("localhost", 27017), proxy_opts) + + # 3. All proxy addresses refuse connection + dead_sock = socket.socket() + dead_sock.bind(("127.0.0.1", 0)) + dead_addr = dead_sock.getsockname() + dead_sock.close() + dead_results = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr)] + + async def dead_async(*_a: Any, **_kw: Any) -> Any: + return dead_results + + replacement_dead: Any = (lambda *_a, **_kw: dead_results) if _IS_SYNC else dead_async + with ( + mock.patch(target, new=replacement_dead), + self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), + ): + await _async_create_connection(("localhost", 27017), proxy_opts) + + async def test_in_process_socks5_server_forwarding(self) -> None: + from test.asynchronous.test_socks5_prose import Socks5Server + + echo_listener = socket.socket() + echo_listener.bind(("127.0.0.1", 0)) + echo_listener.listen(2) + echo_addr = echo_listener.getsockname() + + def serve_echo() -> None: + for _ in range(2): + try: + conn, _ = echo_listener.accept() + except OSError: + return + with conn: + data = conn.recv(64) + if data: + conn.sendall(b"ack:" + data) + + echo_thread = threading.Thread(target=serve_echo, daemon=True) + echo_thread.start() + self.addCleanup(echo_listener.close) + + mapping = {("mapped.example", 12345): echo_addr} + auth_proxy = Socks5Server(0, mapping, ("user", "secret")) + noauth_proxy = Socks5Server(0, mapping) + self.addCleanup(auth_proxy.close) + self.addCleanup(noauth_proxy.close) + + # Test authenticated domain mapping + bidirectional forwarding + sock1 = await _async_create_connection( + ("mapped.example", 12345), + PoolOptions( + connect_timeout=2, + proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "secret"), + ), + ) + try: + sock1.settimeout(2) + sock1.sendall(b"hello") + self.assertEqual(sock1.recv(64), b"ack:hello") + finally: + sock1.close() + + # Test unauthenticated IPv4 connect + bidirectional forwarding + sock2 = await _async_create_connection( + echo_addr, + PoolOptions( + connect_timeout=2, + proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), + ), + ) + try: + sock2.settimeout(2) + sock2.sendall(b"world") + self.assertEqual(sock2.recv(64), b"ack:world") + finally: + sock2.close() + + # Test bad auth and unreachable target against Socks5Server + with self.assertRaises(_Socks5Error): + await _async_create_connection( + ("mapped.example", 12345), + PoolOptions( + connect_timeout=2, + proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "wrong"), + ), + ) + with self.assertRaises(_Socks5Error): + await _async_create_connection( + ("127.0.0.1", 1), + PoolOptions( + connect_timeout=2, + proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), + ), + ) + echo_thread.join(timeout=2) diff --git a/test/test_socks5.py b/test/test_socks5.py index ee3c1118de..84f6d5c356 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -573,3 +573,170 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: finally: parent_client.close() kv_client.close() + + def test_socks5_helper_edge_cases(self) -> None: + from pymongo.common import validate_proxy_host, validate_proxy_port + from pymongo.pool_shared import ( + _proxy_deadline, + _proxy_remaining, + _socks5_check_connect_reply, + _socks5_request, + _Socks5Timeout, + ) + + self.assertIsNone(validate_proxy_host("proxyHost", None)) + self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") + with self.assertRaises(ValueError): + validate_proxy_host("proxyHost", "[not-ipv6]") + with self.assertRaises(ValueError): + validate_proxy_host("proxyHost", "bad/host") + self.assertIsNone(validate_proxy_port("proxyPort", None)) + with self.assertRaises(TypeError): + validate_proxy_port("proxyPort", True) + + self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) + self.assertIsNone(_proxy_remaining(None)) + with self.assertRaises(_Socks5Timeout): + _proxy_remaining(0.0) + with ( + mock.patch("pymongo.pool_shared._csot.remaining", return_value=0.0), + self.assertRaises(_Socks5Timeout), + ): + _proxy_deadline(PoolOptions(connect_timeout=1)) + + with self.assertRaisesRegex(_Socks5Error, "invalid domain name length"): + _socks5_request(("a" * 256, 27017)) + with self.assertRaisesRegex(_Socks5Error, "invalid domain name"): + _socks5_request((".münich.example", 27017)) + with self.assertRaisesRegex(_Socks5Error, "destination port is required"): + _socks5_request(("localhost", None)) # type: ignore[arg-type] + with self.assertRaisesRegex(_Socks5Error, "invalid reserved byte"): + _socks5_check_connect_reply(b"\x05\x00\x01\x01") + with self.assertRaisesRegex(_Socks5Error, "invalid address type"): + _socks5_check_connect_reply(b"\x05\x00\x00\x09") + + def test_proxy_connection_error_paths(self) -> None: + proxy_opts = PoolOptions( + connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080) + ) + target = ( + "pymongo.pool_shared.socket.getaddrinfo" + if _IS_SYNC + else "pymongo.pool_shared._getaddrinfo" + ) + + # 1. getaddrinfo raises OSError + def raise_gai(*_a: Any, **_kw: Any) -> Any: + raise socket.gaierror("resolution failed") + + with ( + mock.patch(target, side_effect=raise_gai), + self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), + ): + _create_connection(("localhost", 27017), proxy_opts) + + # 2. getaddrinfo returns empty list + def empty_async(*_a: Any, **_kw: Any) -> Any: + return [] + + replacement_empty: Any = (lambda *_a, **_kw: []) if _IS_SYNC else empty_async + with ( + mock.patch(target, new=replacement_empty), + self.assertRaisesRegex(_Socks5Error, "returned no addresses"), + ): + _create_connection(("localhost", 27017), proxy_opts) + + # 3. All proxy addresses refuse connection + dead_sock = socket.socket() + dead_sock.bind(("127.0.0.1", 0)) + dead_addr = dead_sock.getsockname() + dead_sock.close() + dead_results = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr)] + + def dead_async(*_a: Any, **_kw: Any) -> Any: + return dead_results + + replacement_dead: Any = (lambda *_a, **_kw: dead_results) if _IS_SYNC else dead_async + with ( + mock.patch(target, new=replacement_dead), + self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), + ): + _create_connection(("localhost", 27017), proxy_opts) + + def test_in_process_socks5_server_forwarding(self) -> None: + from test.test_socks5_prose import Socks5Server + + echo_listener = socket.socket() + echo_listener.bind(("127.0.0.1", 0)) + echo_listener.listen(2) + echo_addr = echo_listener.getsockname() + + def serve_echo() -> None: + for _ in range(2): + try: + conn, _ = echo_listener.accept() + except OSError: + return + with conn: + data = conn.recv(64) + if data: + conn.sendall(b"ack:" + data) + + echo_thread = threading.Thread(target=serve_echo, daemon=True) + echo_thread.start() + self.addCleanup(echo_listener.close) + + mapping = {("mapped.example", 12345): echo_addr} + auth_proxy = Socks5Server(0, mapping, ("user", "secret")) + noauth_proxy = Socks5Server(0, mapping) + self.addCleanup(auth_proxy.close) + self.addCleanup(noauth_proxy.close) + + # Test authenticated domain mapping + bidirectional forwarding + sock1 = _create_connection( + ("mapped.example", 12345), + PoolOptions( + connect_timeout=2, + proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "secret"), + ), + ) + try: + sock1.settimeout(2) + sock1.sendall(b"hello") + self.assertEqual(sock1.recv(64), b"ack:hello") + finally: + sock1.close() + + # Test unauthenticated IPv4 connect + bidirectional forwarding + sock2 = _create_connection( + echo_addr, + PoolOptions( + connect_timeout=2, + proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), + ), + ) + try: + sock2.settimeout(2) + sock2.sendall(b"world") + self.assertEqual(sock2.recv(64), b"ack:world") + finally: + sock2.close() + + # Test bad auth and unreachable target against Socks5Server + with self.assertRaises(_Socks5Error): + _create_connection( + ("mapped.example", 12345), + PoolOptions( + connect_timeout=2, + proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "wrong"), + ), + ) + with self.assertRaises(_Socks5Error): + _create_connection( + ("127.0.0.1", 1), + PoolOptions( + connect_timeout=2, + proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), + ), + ) + echo_thread.join(timeout=2) From 6f933b2c488d0a419034dd576d1a57017dfa4166 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 10:20:26 +0000 Subject: [PATCH 05/16] PYTHON-2943 Fix SOCKS5 CI failures Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/changelog.rst | 4 +- pymongo/pool_shared.py | 31 +++-- test/asynchronous/test_socks5.py | 160 +++++++++++++++++++++---- test/asynchronous/test_socks5_prose.py | 19 +-- test/test_socks5.py | 160 +++++++++++++++++++++---- test/test_socks5_prose.py | 19 +-- 6 files changed, 293 insertions(+), 100 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index c4ad313ca5..b00b46074b 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,7 +8,9 @@ Changes in Version 4.20.0 (2026/XX/XX) Encryption KMS servers through a SOCKS5 proxy via the ``proxyHost``, ``proxyPort``, ``proxyUsername``, and ``proxyPassword`` URI and keyword options on :class:`~pymongo.synchronous.mongo_client.MongoClient` and - :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient`. + Asynchronous proxy connections support cancellation during connection + establishment and negotiation (`PYTHON-2943`_). - The client handshake metadata now reports a detected coding agent in ``client.env.agent`` (`PYTHON-5929`_). diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index fe4e1a30e3..bb3d3f150f 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -746,9 +746,8 @@ async def _async_socks5_read_exact( data = bytearray() loop = asyncio.get_running_loop() while len(data) < size: - chunk = await asyncio.wait_for( - loop.sock_recv(sock, size - len(data)), timeout=_proxy_remaining(deadline) - ) + async with asyncio.timeout(_proxy_remaining(deadline)): + chunk = await loop.sock_recv(sock, size - len(data)) if not chunk: raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") data.extend(chunk) @@ -760,19 +759,17 @@ async def _async_socks5_handshake( ) -> None: loop = asyncio.get_running_loop() greeting, allowed = _socks5_greeting(proxy) - await asyncio.wait_for(loop.sock_sendall(sock, greeting), timeout=_proxy_remaining(deadline)) + async with asyncio.timeout(_proxy_remaining(deadline)): + await loop.sock_sendall(sock, greeting) method = _socks5_check_greeting_reply( await _async_socks5_read_exact(sock, 2, deadline), allowed ) if method == 0x02: - await asyncio.wait_for( - loop.sock_sendall(sock, _socks5_auth_request(proxy)), - timeout=_proxy_remaining(deadline), - ) + async with asyncio.timeout(_proxy_remaining(deadline)): + await loop.sock_sendall(sock, _socks5_auth_request(proxy)) _socks5_check_auth_reply(await _async_socks5_read_exact(sock, 2, deadline)) - await asyncio.wait_for( - loop.sock_sendall(sock, _socks5_request(address)), timeout=_proxy_remaining(deadline) - ) + async with asyncio.timeout(_proxy_remaining(deadline)): + await loop.sock_sendall(sock, _socks5_request(address)) atyp = _socks5_check_connect_reply(await _async_socks5_read_exact(sock, 4, deadline)) if atyp == 0x01: bound_size = 6 @@ -791,9 +788,10 @@ async def _async_create_proxy_connection( if socket.has_ipv6 and proxy.host != "localhost": family = socket.AF_UNSPEC try: - remaining = _proxy_remaining(deadline) - getaddrinfo = _getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) - results = await asyncio.wait_for(getaddrinfo, timeout=remaining) + async with asyncio.timeout(_proxy_remaining(deadline)): + results = await _getaddrinfo( + proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM + ) except (asyncio.TimeoutError, socket.timeout) as exc: raise _Socks5Timeout("timed out") from exc except OSError as exc: @@ -813,9 +811,8 @@ async def _async_create_proxy_connection( sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) _set_keepalive_times(sock) sock.setblocking(False) - await asyncio.wait_for( - loop.sock_connect(sock, sockaddr), timeout=_proxy_remaining(deadline) - ) + async with asyncio.timeout(_proxy_remaining(deadline)): + await loop.sock_connect(sock, sockaddr) connected = True await _async_socks5_handshake(sock, address, proxy, deadline) post_timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index fca790e69d..ebcf0f2db6 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -22,6 +22,8 @@ import ssl import sys import threading +from collections.abc import Iterator +from contextlib import contextmanager from types import SimpleNamespace from typing import Any, Optional, cast from unittest import mock @@ -173,6 +175,31 @@ def close(self) -> None: self.thread.join(timeout=3) +@contextmanager +def _refuse_connection(address: tuple[str, int]) -> Iterator[None]: + if _IS_SYNC: + original_connect = socket.socket.connect + + def connect(sock: socket.socket, target: tuple[str, int]) -> None: + if target == address: + raise ConnectionRefusedError("connection refused") + original_connect(sock, target) + + with mock.patch.object(socket.socket, "connect", new=connect): + yield + else: + loop = asyncio.get_running_loop() + original_sock_connect = loop.sock_connect + + async def sock_connect(sock: socket.socket, target: tuple[str, int]) -> None: + if target == address: + raise ConnectionRefusedError("connection refused") + await original_sock_connect(sock, target) + + with mock.patch.object(loop, "sock_connect", new=sock_connect): + yield + + class TestSocks5(AsyncPyMongoTestCase): async def _connect( self, @@ -331,12 +358,32 @@ async def test_timeout_covers_the_entire_handshake(self) -> None: @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the asynchronous driver") async def test_cancellation_closes_proxy_socket(self) -> None: server = _Socks5Server(stall=True) + self.addCleanup(server.close) task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) - await asyncio.to_thread(server.started.wait, 2) + self.assertTrue(await asyncio.to_thread(server.started.wait, 2)) task.cancel() with self.assertRaises(asyncio.CancelledError): await task - server.close() + + @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the asynchronous driver") + async def test_cancellation_when_proxy_send_completes_closes_socket(self) -> None: + server = _Socks5Server(stall=True) + self.addCleanup(server.close) + loop = asyncio.get_running_loop() + original_sendall = loop.sock_sendall + sockets: list[socket.socket] = [] + + async def cancel_after_send(sock: socket.socket, data: bytes) -> None: + sockets.append(sock) + await original_sendall(sock, data) + task.cancel() + + with mock.patch.object(loop, "sock_sendall", new=cancel_after_send): + task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) + with self.assertRaises(asyncio.CancelledError): + await task + self.assertEqual(len(sockets), 1) + self.assertEqual(sockets[0].fileno(), -1) async def test_tls_sni_targets_destination_not_proxy(self) -> None: listener = socket.socket() @@ -515,10 +562,7 @@ async def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: async def test_multi_address_proxy_fallback_on_connect_error(self) -> None: server = _Socks5Server() self.addCleanup(server.close) - dead_sock = socket.socket() - dead_sock.bind(("127.0.0.1", 0)) - dead_addr = dead_sock.getsockname() - dead_sock.close() + dead_addr = ("127.0.0.1", 1) fake_results = [ (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), @@ -534,7 +578,7 @@ async def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: return fake_results replacement: Any = (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo - with mock.patch(target, new=replacement): + with mock.patch(target, new=replacement), _refuse_connection(dead_addr): sock = await self._connect(server, ("localhost", 27017)) sock.close() @@ -625,7 +669,6 @@ async def test_proxy_connection_error_paths(self) -> None: else "pymongo.pool_shared._getaddrinfo" ) - # 1. getaddrinfo raises OSError def raise_gai(*_a: Any, **_kw: Any) -> Any: raise socket.gaierror("resolution failed") @@ -635,7 +678,6 @@ def raise_gai(*_a: Any, **_kw: Any) -> Any: ): await _async_create_connection(("localhost", 27017), proxy_opts) - # 2. getaddrinfo returns empty list async def empty_async(*_a: Any, **_kw: Any) -> Any: return [] @@ -646,11 +688,7 @@ async def empty_async(*_a: Any, **_kw: Any) -> Any: ): await _async_create_connection(("localhost", 27017), proxy_opts) - # 3. All proxy addresses refuse connection - dead_sock = socket.socket() - dead_sock.bind(("127.0.0.1", 0)) - dead_addr = dead_sock.getsockname() - dead_sock.close() + dead_addr = ("127.0.0.1", 1) dead_results = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr)] async def dead_async(*_a: Any, **_kw: Any) -> Any: @@ -659,10 +697,66 @@ async def dead_async(*_a: Any, **_kw: Any) -> Any: replacement_dead: Any = (lambda *_a, **_kw: dead_results) if _IS_SYNC else dead_async with ( mock.patch(target, new=replacement_dead), + _refuse_connection(dead_addr), self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), ): await _async_create_connection(("localhost", 27017), proxy_opts) + async def test_prose_client_options(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + from pymongo.errors import OperationFailure + from test.asynchronous import test_socks5_prose + + async def command(name: str) -> None: + if name == "nonexistent_socks5_test_command": + raise OperationFailure("unsupported command") + + async def close() -> None: + pass + + fake_client = SimpleNamespace(admin=SimpleNamespace(command=command), close=close) + case = test_socks5_prose.TestSocks5Prose() + case.credentials = {"username": "dbuser", "password": "dbpass", "authSource": "admin"} + listener = test_socks5_prose._CommandListener() + for tls in (False, True): + for use_uri in (False, True): + for event_listener in (None, listener): + with self.subTest(tls=tls, use_uri=use_uri, listener=event_listener): + with ( + mock.patch.object(test_socks5_prose.async_client_context, "tls", tls), + mock.patch.object( + test_socks5_prose, "AsyncMongoClient", return_value=fake_client + ) as factory, + ): + await case._run_case( + "localhost:12345", + 1080, + ("proxyuser", "proxypass"), + True, + use_uri, + event_listener, + ) + factory.assert_called_once() + args, kwargs = factory.call_args + client = AsyncMongoClient(*args, connect=False, **kwargs) + try: + self.assertEqual( + client.options.pool_options._proxy_options, + _ProxyOptions("localhost", 1080, "proxyuser", "proxypass"), + ) + self.assertEqual(client.options.direct_connection, True) + self.assertEqual(client.options.server_selection_timeout, 2.5) + self.assertEqual( + client.options.pool_options._ssl_context is not None, tls + ) + self.assertEqual(kwargs["authSource"], "admin") + if event_listener is None: + self.assertNotIn("event_listeners", kwargs) + else: + self.assertEqual(kwargs["event_listeners"], [event_listener]) + finally: + await client.close() + async def test_in_process_socks5_server_forwarding(self) -> None: from test.asynchronous.test_socks5_prose import Socks5Server @@ -684,6 +778,7 @@ def serve_echo() -> None: echo_thread = threading.Thread(target=serve_echo, daemon=True) echo_thread.start() + self.addCleanup(echo_thread.join, 2) self.addCleanup(echo_listener.close) mapping = {("mapped.example", 12345): echo_addr} @@ -692,7 +787,6 @@ def serve_echo() -> None: self.addCleanup(auth_proxy.close) self.addCleanup(noauth_proxy.close) - # Test authenticated domain mapping + bidirectional forwarding sock1 = await _async_create_connection( ("mapped.example", 12345), PoolOptions( @@ -701,13 +795,19 @@ def serve_echo() -> None: ), ) try: - sock1.settimeout(2) - sock1.sendall(b"hello") - self.assertEqual(sock1.recv(64), b"ack:hello") + if _IS_SYNC: + sock1.settimeout(2) + sock1.sendall(b"hello") + response = sock1.recv(64) + else: + sock1.setblocking(False) + loop = asyncio.get_running_loop() + await loop.sock_sendall(sock1, b"hello") + response = await asyncio.wait_for(loop.sock_recv(sock1, 64), timeout=2) + self.assertEqual(response, b"ack:hello") finally: sock1.close() - # Test unauthenticated IPv4 connect + bidirectional forwarding sock2 = await _async_create_connection( echo_addr, PoolOptions( @@ -716,13 +816,19 @@ def serve_echo() -> None: ), ) try: - sock2.settimeout(2) - sock2.sendall(b"world") - self.assertEqual(sock2.recv(64), b"ack:world") + if _IS_SYNC: + sock2.settimeout(2) + sock2.sendall(b"world") + response = sock2.recv(64) + else: + sock2.setblocking(False) + loop = asyncio.get_running_loop() + await loop.sock_sendall(sock2, b"world") + response = await asyncio.wait_for(loop.sock_recv(sock2, 64), timeout=2) + self.assertEqual(response, b"ack:world") finally: sock2.close() - # Test bad auth and unreachable target against Socks5Server with self.assertRaises(_Socks5Error): await _async_create_connection( ("mapped.example", 12345), @@ -731,7 +837,12 @@ def serve_echo() -> None: proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "wrong"), ), ) - with self.assertRaises(_Socks5Error): + with ( + mock.patch( + "socket.create_connection", side_effect=ConnectionRefusedError("connection refused") + ), + self.assertRaisesRegex(_Socks5Error, "connection refused"), + ): await _async_create_connection( ("127.0.0.1", 1), PoolOptions( @@ -739,4 +850,3 @@ def serve_echo() -> None: proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), ), ) - echo_thread.join(timeout=2) diff --git a/test/asynchronous/test_socks5_prose.py b/test/asynchronous/test_socks5_prose.py index dca5e2f5c0..d49f8f6fa7 100644 --- a/test/asynchronous/test_socks5_prose.py +++ b/test/asynchronous/test_socks5_prose.py @@ -222,7 +222,6 @@ async def _run_case( tls_options = dict(TLS_OPTIONS) if async_client_context.tls else {} options = dict(tls_options) options.update(self.credentials) - options.update(proxy_options) options["serverSelectionTimeoutMS"] = 2500 if hosts == "localhost:12345": options["directConnection"] = True @@ -232,22 +231,10 @@ async def _run_case( uri_options = "&".join( f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() ) - uri_options += "&serverSelectionTimeoutMS=2500" - if hosts == "localhost:12345": - uri_options += "&directConnection=true" - if tls_options: - uri_options += "&" + "&".join( - f"{key}={quote_plus(str(value))}" for key, value in tls_options.items() - ) - auth = "" - if self.credentials: - auth = f"{quote_plus(self.credentials['username'])}:{quote_plus(self.credentials['password'])}@" - client = AsyncMongoClient( - f"mongodb://{auth}{hosts}/?{uri_options}", - event_listeners=options.get("event_listeners"), - ) + hosts = f"mongodb://{hosts}/?{uri_options}" else: - client = AsyncMongoClient(hosts, **options) + options.update(proxy_options) + client = AsyncMongoClient(hosts, **options) try: if succeeds: await client.admin.command(HelloCompat.CMD) diff --git a/test/test_socks5.py b/test/test_socks5.py index 84f6d5c356..f3bb952f6a 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -22,6 +22,8 @@ import ssl import sys import threading +from collections.abc import Iterator +from contextlib import contextmanager from types import SimpleNamespace from typing import Any, Optional, cast from unittest import mock @@ -173,6 +175,31 @@ def close(self) -> None: self.thread.join(timeout=3) +@contextmanager +def _refuse_connection(address: tuple[str, int]) -> Iterator[None]: + if _IS_SYNC: + original_connect = socket.socket.connect + + def connect(sock: socket.socket, target: tuple[str, int]) -> None: + if target == address: + raise ConnectionRefusedError("connection refused") + original_connect(sock, target) + + with mock.patch.object(socket.socket, "connect", new=connect): + yield + else: + loop = asyncio.get_running_loop() + original_sock_connect = loop.sock_connect + + def sock_connect(sock: socket.socket, target: tuple[str, int]) -> None: + if target == address: + raise ConnectionRefusedError("connection refused") + original_sock_connect(sock, target) + + with mock.patch.object(loop, "sock_connect", new=sock_connect): + yield + + class TestSocks5(PyMongoTestCase): def _connect( self, @@ -331,12 +358,32 @@ def test_timeout_covers_the_entire_handshake(self) -> None: @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the synchronous driver") def test_cancellation_closes_proxy_socket(self) -> None: server = _Socks5Server(stall=True) + self.addCleanup(server.close) task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) - asyncio.to_thread(server.started.wait, 2) + self.assertTrue(asyncio.to_thread(server.started.wait, 2)) task.cancel() with self.assertRaises(asyncio.CancelledError): task - server.close() + + @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the synchronous driver") + def test_cancellation_when_proxy_send_completes_closes_socket(self) -> None: + server = _Socks5Server(stall=True) + self.addCleanup(server.close) + loop = asyncio.get_running_loop() + original_sendall = loop.sock_sendall + sockets: list[socket.socket] = [] + + def cancel_after_send(sock: socket.socket, data: bytes) -> None: + sockets.append(sock) + original_sendall(sock, data) + task.cancel() + + with mock.patch.object(loop, "sock_sendall", new=cancel_after_send): + task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) + with self.assertRaises(asyncio.CancelledError): + task + self.assertEqual(len(sockets), 1) + self.assertEqual(sockets[0].fileno(), -1) def test_tls_sni_targets_destination_not_proxy(self) -> None: listener = socket.socket() @@ -515,10 +562,7 @@ def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: def test_multi_address_proxy_fallback_on_connect_error(self) -> None: server = _Socks5Server() self.addCleanup(server.close) - dead_sock = socket.socket() - dead_sock.bind(("127.0.0.1", 0)) - dead_addr = dead_sock.getsockname() - dead_sock.close() + dead_addr = ("127.0.0.1", 1) fake_results = [ (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), @@ -534,7 +578,7 @@ def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: return fake_results replacement: Any = (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo - with mock.patch(target, new=replacement): + with mock.patch(target, new=replacement), _refuse_connection(dead_addr): sock = self._connect(server, ("localhost", 27017)) sock.close() @@ -625,7 +669,6 @@ def test_proxy_connection_error_paths(self) -> None: else "pymongo.pool_shared._getaddrinfo" ) - # 1. getaddrinfo raises OSError def raise_gai(*_a: Any, **_kw: Any) -> Any: raise socket.gaierror("resolution failed") @@ -635,7 +678,6 @@ def raise_gai(*_a: Any, **_kw: Any) -> Any: ): _create_connection(("localhost", 27017), proxy_opts) - # 2. getaddrinfo returns empty list def empty_async(*_a: Any, **_kw: Any) -> Any: return [] @@ -646,11 +688,7 @@ def empty_async(*_a: Any, **_kw: Any) -> Any: ): _create_connection(("localhost", 27017), proxy_opts) - # 3. All proxy addresses refuse connection - dead_sock = socket.socket() - dead_sock.bind(("127.0.0.1", 0)) - dead_addr = dead_sock.getsockname() - dead_sock.close() + dead_addr = ("127.0.0.1", 1) dead_results = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr)] def dead_async(*_a: Any, **_kw: Any) -> Any: @@ -659,10 +697,66 @@ def dead_async(*_a: Any, **_kw: Any) -> Any: replacement_dead: Any = (lambda *_a, **_kw: dead_results) if _IS_SYNC else dead_async with ( mock.patch(target, new=replacement_dead), + _refuse_connection(dead_addr), self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), ): _create_connection(("localhost", 27017), proxy_opts) + def test_prose_client_options(self) -> None: + from pymongo.errors import OperationFailure + from pymongo.synchronous.mongo_client import MongoClient + from test import test_socks5_prose + + def command(name: str) -> None: + if name == "nonexistent_socks5_test_command": + raise OperationFailure("unsupported command") + + def close() -> None: + pass + + fake_client = SimpleNamespace(admin=SimpleNamespace(command=command), close=close) + case = test_socks5_prose.TestSocks5Prose() + case.credentials = {"username": "dbuser", "password": "dbpass", "authSource": "admin"} + listener = test_socks5_prose._CommandListener() + for tls in (False, True): + for use_uri in (False, True): + for event_listener in (None, listener): + with self.subTest(tls=tls, use_uri=use_uri, listener=event_listener): + with ( + mock.patch.object(test_socks5_prose.client_context, "tls", tls), + mock.patch.object( + test_socks5_prose, "MongoClient", return_value=fake_client + ) as factory, + ): + case._run_case( + "localhost:12345", + 1080, + ("proxyuser", "proxypass"), + True, + use_uri, + event_listener, + ) + factory.assert_called_once() + args, kwargs = factory.call_args + client = MongoClient(*args, connect=False, **kwargs) + try: + self.assertEqual( + client.options.pool_options._proxy_options, + _ProxyOptions("localhost", 1080, "proxyuser", "proxypass"), + ) + self.assertEqual(client.options.direct_connection, True) + self.assertEqual(client.options.server_selection_timeout, 2.5) + self.assertEqual( + client.options.pool_options._ssl_context is not None, tls + ) + self.assertEqual(kwargs["authSource"], "admin") + if event_listener is None: + self.assertNotIn("event_listeners", kwargs) + else: + self.assertEqual(kwargs["event_listeners"], [event_listener]) + finally: + client.close() + def test_in_process_socks5_server_forwarding(self) -> None: from test.test_socks5_prose import Socks5Server @@ -684,6 +778,7 @@ def serve_echo() -> None: echo_thread = threading.Thread(target=serve_echo, daemon=True) echo_thread.start() + self.addCleanup(echo_thread.join, 2) self.addCleanup(echo_listener.close) mapping = {("mapped.example", 12345): echo_addr} @@ -692,7 +787,6 @@ def serve_echo() -> None: self.addCleanup(auth_proxy.close) self.addCleanup(noauth_proxy.close) - # Test authenticated domain mapping + bidirectional forwarding sock1 = _create_connection( ("mapped.example", 12345), PoolOptions( @@ -701,13 +795,19 @@ def serve_echo() -> None: ), ) try: - sock1.settimeout(2) - sock1.sendall(b"hello") - self.assertEqual(sock1.recv(64), b"ack:hello") + if _IS_SYNC: + sock1.settimeout(2) + sock1.sendall(b"hello") + response = sock1.recv(64) + else: + sock1.setblocking(False) + loop = asyncio.get_running_loop() + loop.sock_sendall(sock1, b"hello") + response = asyncio.wait_for(loop.sock_recv(sock1, 64), timeout=2) + self.assertEqual(response, b"ack:hello") finally: sock1.close() - # Test unauthenticated IPv4 connect + bidirectional forwarding sock2 = _create_connection( echo_addr, PoolOptions( @@ -716,13 +816,19 @@ def serve_echo() -> None: ), ) try: - sock2.settimeout(2) - sock2.sendall(b"world") - self.assertEqual(sock2.recv(64), b"ack:world") + if _IS_SYNC: + sock2.settimeout(2) + sock2.sendall(b"world") + response = sock2.recv(64) + else: + sock2.setblocking(False) + loop = asyncio.get_running_loop() + loop.sock_sendall(sock2, b"world") + response = asyncio.wait_for(loop.sock_recv(sock2, 64), timeout=2) + self.assertEqual(response, b"ack:world") finally: sock2.close() - # Test bad auth and unreachable target against Socks5Server with self.assertRaises(_Socks5Error): _create_connection( ("mapped.example", 12345), @@ -731,7 +837,12 @@ def serve_echo() -> None: proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "wrong"), ), ) - with self.assertRaises(_Socks5Error): + with ( + mock.patch( + "socket.create_connection", side_effect=ConnectionRefusedError("connection refused") + ), + self.assertRaisesRegex(_Socks5Error, "connection refused"), + ): _create_connection( ("127.0.0.1", 1), PoolOptions( @@ -739,4 +850,3 @@ def serve_echo() -> None: proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), ), ) - echo_thread.join(timeout=2) diff --git a/test/test_socks5_prose.py b/test/test_socks5_prose.py index 3c0a28ea44..9e4680d3b5 100644 --- a/test/test_socks5_prose.py +++ b/test/test_socks5_prose.py @@ -222,7 +222,6 @@ def _run_case( tls_options = dict(TLS_OPTIONS) if client_context.tls else {} options = dict(tls_options) options.update(self.credentials) - options.update(proxy_options) options["serverSelectionTimeoutMS"] = 2500 if hosts == "localhost:12345": options["directConnection"] = True @@ -232,22 +231,10 @@ def _run_case( uri_options = "&".join( f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() ) - uri_options += "&serverSelectionTimeoutMS=2500" - if hosts == "localhost:12345": - uri_options += "&directConnection=true" - if tls_options: - uri_options += "&" + "&".join( - f"{key}={quote_plus(str(value))}" for key, value in tls_options.items() - ) - auth = "" - if self.credentials: - auth = f"{quote_plus(self.credentials['username'])}:{quote_plus(self.credentials['password'])}@" - client = MongoClient( - f"mongodb://{auth}{hosts}/?{uri_options}", - event_listeners=options.get("event_listeners"), - ) + hosts = f"mongodb://{hosts}/?{uri_options}" else: - client = MongoClient(hosts, **options) + options.update(proxy_options) + client = MongoClient(hosts, **options) try: if succeeds: client.admin.command(HelloCompat.CMD) From 2d03b2c2bd5fd163629db8b50363e52ea757680d Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 12:35:56 +0000 Subject: [PATCH 06/16] PYTHON-2943 Retrigger PR synchronization Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> From 38502f3b7dfbc9704e73568e997493b73aa2ea72 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 14:58:27 +0000 Subject: [PATCH 07/16] PYTHON-2943 Address SOCKS5 review improvements Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pymongo/asynchronous/mongo_client.py | 13 +- pymongo/client_options.py | 4 +- pymongo/pool_options.py | 3 +- pymongo/pool_shared.py | 232 ++++++++++++-------- pymongo/synchronous/mongo_client.py | 13 +- pymongo/uri_parser_shared.py | 8 +- test/asynchronous/test_socks5.py | 158 ++------------ test/asynchronous/test_socks5_prose.py | 133 +----------- test/socks5_helpers.py | 287 +++++++++++++++++++++++++ test/test_socks5.py | 158 ++------------ test/test_socks5_prose.py | 133 +----------- 11 files changed, 498 insertions(+), 644 deletions(-) create mode 100644 test/socks5_helpers.py diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 959697e51b..a115d1f66c 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -100,6 +100,7 @@ UpdateOne, _Op, ) +from pymongo.pool_options import _ProxyOptions from pymongo.read_preferences import ReadPreference, _ServerMode from pymongo.results import ClientBulkWriteResult from pymongo.server_description import ServerDescription @@ -674,6 +675,10 @@ def validator(host: str) -> bool: .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.20 + Added the ``proxyHost``, ``proxyPort``, ``proxyUsername``, and + ``proxyPassword`` keyword arguments and URI options. + .. versionchanged:: 4.19 Added the ``srv_host_validator`` keyword argument. @@ -1125,9 +1130,6 @@ def _validate_kwargs_and_update_opts( keyword_opts = common._CaseInsensitiveDictionary( dict(common.validate(keyword_opts.cased_key(k), v) for k, v in keyword_opts.items()) ) - for name in ("proxyusername", "proxypassword"): - if keyword_opts.get(name) == "": - keyword_opts.pop(name) # Override connection string options with kwarg options. opts.update(keyword_opts) return opts @@ -1386,12 +1388,15 @@ def options(self) -> ClientOptions: """ return self._options - def eq_props(self) -> tuple[tuple[_Address, ...], Optional[str], Optional[str], str]: + def eq_props( + self, + ) -> tuple[tuple[_Address, ...], Optional[str], Optional[str], str, Optional[_ProxyOptions]]: return ( tuple(sorted(self._resolve_srv_info["seeds"])), self._options.replica_set_name, self._resolve_srv_info["fqdn"], self._resolve_srv_info["srv_service_name"], + self._options.pool_options._proxy_options, ) def __eq__(self, other: Any) -> bool: diff --git a/pymongo/client_options.py b/pymongo/client_options.py index 6fffa30ccf..ed95e00646 100644 --- a/pymongo/client_options.py +++ b/pymongo/client_options.py @@ -27,7 +27,7 @@ from pymongo.compression_support import CompressionSettings from pymongo.errors import ConfigurationError from pymongo.monitoring import _EventListener, _EventListeners -from pymongo.pool_options import PoolOptions, _ProxyOptions +from pymongo.pool_options import _DEFAULT_PROXY_PORT, PoolOptions, _ProxyOptions from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ( _ServerMode, @@ -178,7 +178,7 @@ def _parse_pool_options( proxy_options = ( _ProxyOptions( proxy_host, - options.get("proxyport") or 1080, + options.get("proxyport") or _DEFAULT_PROXY_PORT, options.get("proxyusername"), options.get("proxypassword"), ) diff --git a/pymongo/pool_options.py b/pymongo/pool_options.py index c86c6fe526..b79a344822 100644 --- a/pymongo/pool_options.py +++ b/pymongo/pool_options.py @@ -103,6 +103,7 @@ RUNTIME_NAME_DOCKER = "docker" ORCHESTRATOR_NAME_K8S = "kubernetes" +_DEFAULT_PROXY_PORT = 1080 def get_container_env_info() -> dict[str, str]: @@ -375,7 +376,7 @@ class _ProxyOptions: """Immutable SOCKS5 proxy connection settings.""" host: str - port: int = 1080 + port: int = _DEFAULT_PROXY_PORT username: Optional[str] = None password: Optional[str] = field(default=None, repr=False) diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index bb3d3f150f..9306a1ce62 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -76,6 +76,36 @@ from pymongo.pyopenssl_context import _sslConn from pymongo.typings import _Address, _AgnosticClientSession, _AgnosticMongoClient +_SOCKS5_VERSION = 0x05 +_SOCKS5_AUTH_VERSION = 0x01 +_SOCKS5_AUTH_NO_AUTH = 0x00 +_SOCKS5_AUTH_USER_PASS = 0x02 +_SOCKS5_AUTH_NO_ACCEPTABLE = 0xFF +_SOCKS5_AUTH_SUCCESS = 0x00 +_SOCKS5_RESERVED = 0x00 +_SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS = 1 +_SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS = 2 +_SOCKS5_CMD_CONNECT = 0x01 +_SOCKS5_REPLY_GENERAL_FAILURE = 0x01 +_SOCKS5_REPLY_RULESET_DENIED = 0x02 +_SOCKS5_REPLY_NETWORK_UNREACHABLE = 0x03 +_SOCKS5_REPLY_HOST_UNREACHABLE = 0x04 +_SOCKS5_REPLY_CONNECTION_REFUSED = 0x05 +_SOCKS5_REPLY_TTL_EXPIRED = 0x06 +_SOCKS5_REPLY_COMMAND_UNSUPPORTED = 0x07 +_SOCKS5_REPLY_ADDRESS_UNSUPPORTED = 0x08 +_SOCKS5_ATYP_IPV4 = 0x01 +_SOCKS5_ATYP_DOMAIN = 0x03 +_SOCKS5_ATYP_IPV6 = 0x04 +_SOCKS5_IPV4_BOUND_LEN = 6 +_SOCKS5_IPV6_BOUND_LEN = 18 +_SOCKS5_PORT_LEN = 2 +_SOCKS5_GREETING_REPLY_LEN = 2 +_SOCKS5_AUTH_REPLY_LEN = 2 +_SOCKS5_CONNECT_REPLY_LEN = 4 +_SOCKS5_DOMAIN_LENGTH_LEN = 1 +_SOCKS5_MAX_STR_LEN = 255 + # Flags recording the counters a checkout has incremented, so a failed # checkout can restore them exactly once (PYTHON-6136). _UNDO_OPERATION_COUNT = 1 @@ -598,14 +628,14 @@ class PoolState: _SOCKS5_REPLY_REASONS = { - 0x01: "general SOCKS server failure", - 0x02: "connection not allowed by ruleset", - 0x03: "network unreachable", - 0x04: "host unreachable", - 0x05: "connection refused", - 0x06: "TTL expired", - 0x07: "command not supported", - 0x08: "address type not supported", + _SOCKS5_REPLY_GENERAL_FAILURE: "general SOCKS server failure", + _SOCKS5_REPLY_RULESET_DENIED: "connection not allowed by ruleset", + _SOCKS5_REPLY_NETWORK_UNREACHABLE: "network unreachable", + _SOCKS5_REPLY_HOST_UNREACHABLE: "host unreachable", + _SOCKS5_REPLY_CONNECTION_REFUSED: "connection refused", + _SOCKS5_REPLY_TTL_EXPIRED: "TTL expired", + _SOCKS5_REPLY_COMMAND_UNSUPPORTED: "command not supported", + _SOCKS5_REPLY_ADDRESS_UNSUPPORTED: "address type not supported", } @@ -628,6 +658,38 @@ def _proxy_deadline(options: PoolOptions) -> Optional[float]: return time.monotonic() + timeout +def _new_tcp_socket(af: int, socktype: int, proto: int) -> socket.socket: + # SOCK_CLOEXEC is not available on all platforms or kernels. + try: + sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) + except OSError: + sock = socket.socket(af, socktype, proto) + _set_non_inheritable_non_atomic(sock.fileno()) + return sock + + +def _configure_tcp_socket(sock: socket.socket, *, non_blocking: bool = False) -> None: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + _set_keepalive_times(sock) + if non_blocking: + sock.setblocking(False) + + +def _proxy_socket_timeout(options: PoolOptions) -> Optional[float]: + return _csot.remaining() if _csot.get_timeout() else options.connect_timeout + + +def _socks5_bound_address_size(atyp: int, domain_length: Optional[int] = None) -> int: + if atyp == _SOCKS5_ATYP_IPV4: + return _SOCKS5_IPV4_BOUND_LEN + if atyp == _SOCKS5_ATYP_IPV6: + return _SOCKS5_IPV6_BOUND_LEN + if domain_length is None: + raise _Socks5Error("SOCKS5 connect failed: missing bound domain length") + return domain_length + _SOCKS5_PORT_LEN + + def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: if deadline is None: return None @@ -639,16 +701,28 @@ def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: def _socks5_greeting(proxy: _ProxyOptions) -> tuple[bytes, tuple[int, ...]]: if proxy.username is not None and proxy.password is not None: - return b"\x05\x02\x00\x02", (0x00, 0x02) - return b"\x05\x01\x00", (0x00,) + return bytes( + ( + _SOCKS5_VERSION, + _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + ), ( + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + return bytes( + (_SOCKS5_VERSION, _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, _SOCKS5_AUTH_NO_AUTH) + ), (_SOCKS5_AUTH_NO_AUTH,) def _socks5_check_greeting_reply(data: bytes, allowed: tuple[int, ...]) -> int: version, method = data - if version != 0x05: + if version != _SOCKS5_VERSION: raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") if method not in allowed: - if method == 0xFF: + if method == _SOCKS5_AUTH_NO_ACCEPTABLE: raise _Socks5Error( "SOCKS5 initial handshake failed: no acceptable authentication methods" ) @@ -663,12 +737,14 @@ def _socks5_auth_request(proxy: _ProxyOptions) -> bytes: assert proxy.password is not None username = proxy.username.encode("utf-8") password = proxy.password.encode("utf-8") - return bytes((0x01, len(username))) + username + bytes((len(password),)) + password + return ( + bytes((_SOCKS5_AUTH_VERSION, len(username))) + username + bytes((len(password),)) + password + ) def _socks5_check_auth_reply(data: bytes) -> None: auth_version, status = data - if auth_version != 0x01 or status != 0x00: + if auth_version != _SOCKS5_AUTH_VERSION or status != _SOCKS5_AUTH_SUCCESS: raise _Socks5Error("SOCKS5 authentication failed") @@ -681,26 +757,32 @@ def _socks5_request(address: _Address) -> bytes: domain = host.encode("ascii") if host.isascii() else host.encode("idna") except UnicodeError as exc: raise _Socks5Error(f"SOCKS5 connect failed: invalid domain name: {exc}") from exc - if not 1 <= len(domain) <= 255: + if not 1 <= len(domain) <= _SOCKS5_MAX_STR_LEN: raise _Socks5Error("SOCKS5 connect failed: invalid domain name length") from None - encoded_host = b"\x03" + bytes((len(domain),)) + domain + encoded_host = bytes((_SOCKS5_ATYP_DOMAIN, len(domain))) + domain else: - encoded_host = (b"\x01" if ip.version == 4 else b"\x04") + ip.packed + encoded_host = ( + bytes((_SOCKS5_ATYP_IPV4 if ip.version == 4 else _SOCKS5_ATYP_IPV6,)) + ip.packed + ) if port is None: raise _Socks5Error("SOCKS5 connect failed: destination port is required") - return b"\x05\x01\x00" + encoded_host + port.to_bytes(2, "big") + return ( + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) + + encoded_host + + port.to_bytes(_SOCKS5_PORT_LEN, "big") + ) def _socks5_check_connect_reply(data: bytes) -> int: version, reply, reserved, atyp = data - if version != 0x05: + if version != _SOCKS5_VERSION: raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") - if reserved != 0x00: + if reserved != _SOCKS5_RESERVED: raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") if reply: reason = _SOCKS5_REPLY_REASONS.get(reply, f"unknown error {reply:#04x}") raise _Socks5Error(f"SOCKS5 connect failed: {reason}") - if atyp not in (0x01, 0x03, 0x04): + if atyp not in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") return atyp @@ -722,21 +804,25 @@ def _socks5_handshake( greeting, allowed = _socks5_greeting(proxy) sock.settimeout(_proxy_remaining(deadline)) sock.sendall(greeting) - method = _socks5_check_greeting_reply(_socks5_read_exact(sock, 2, deadline), allowed) - if method == 0x02: + method = _socks5_check_greeting_reply( + _socks5_read_exact(sock, _SOCKS5_GREETING_REPLY_LEN, deadline), allowed + ) + if method == _SOCKS5_AUTH_USER_PASS: sock.settimeout(_proxy_remaining(deadline)) sock.sendall(_socks5_auth_request(proxy)) - _socks5_check_auth_reply(_socks5_read_exact(sock, 2, deadline)) + _socks5_check_auth_reply(_socks5_read_exact(sock, _SOCKS5_AUTH_REPLY_LEN, deadline)) sock.settimeout(_proxy_remaining(deadline)) sock.sendall(_socks5_request(address)) - atyp = _socks5_check_connect_reply(_socks5_read_exact(sock, 4, deadline)) - if atyp == 0x01: - bound_size = 6 - elif atyp == 0x04: - bound_size = 18 - else: - bound_size = _socks5_read_exact(sock, 1, deadline)[0] + 2 + atyp = _socks5_check_connect_reply( + _socks5_read_exact(sock, _SOCKS5_CONNECT_REPLY_LEN, deadline) + ) + domain_length = ( + _socks5_read_exact(sock, _SOCKS5_DOMAIN_LENGTH_LEN, deadline)[0] + if atyp == _SOCKS5_ATYP_DOMAIN + else None + ) + bound_size = _socks5_bound_address_size(atyp, domain_length) _socks5_read_exact(sock, bound_size, deadline) @@ -762,21 +848,25 @@ async def _async_socks5_handshake( async with asyncio.timeout(_proxy_remaining(deadline)): await loop.sock_sendall(sock, greeting) method = _socks5_check_greeting_reply( - await _async_socks5_read_exact(sock, 2, deadline), allowed + await _async_socks5_read_exact(sock, _SOCKS5_GREETING_REPLY_LEN, deadline), allowed ) - if method == 0x02: + if method == _SOCKS5_AUTH_USER_PASS: async with asyncio.timeout(_proxy_remaining(deadline)): await loop.sock_sendall(sock, _socks5_auth_request(proxy)) - _socks5_check_auth_reply(await _async_socks5_read_exact(sock, 2, deadline)) + _socks5_check_auth_reply( + await _async_socks5_read_exact(sock, _SOCKS5_AUTH_REPLY_LEN, deadline) + ) async with asyncio.timeout(_proxy_remaining(deadline)): await loop.sock_sendall(sock, _socks5_request(address)) - atyp = _socks5_check_connect_reply(await _async_socks5_read_exact(sock, 4, deadline)) - if atyp == 0x01: - bound_size = 6 - elif atyp == 0x04: - bound_size = 18 - else: - bound_size = (await _async_socks5_read_exact(sock, 1, deadline))[0] + 2 + atyp = _socks5_check_connect_reply( + await _async_socks5_read_exact(sock, _SOCKS5_CONNECT_REPLY_LEN, deadline) + ) + domain_length = ( + (await _async_socks5_read_exact(sock, _SOCKS5_DOMAIN_LENGTH_LEN, deadline))[0] + if atyp == _SOCKS5_ATYP_DOMAIN + else None + ) + bound_size = _socks5_bound_address_size(atyp, domain_length) await _async_socks5_read_exact(sock, bound_size, deadline) @@ -800,23 +890,15 @@ async def _async_create_proxy_connection( err: Optional[OSError] = None loop = asyncio.get_running_loop() for af, socktype, proto, _, sockaddr in results: - try: - sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) - except OSError: - sock = socket.socket(af, socktype, proto) - _set_non_inheritable_non_atomic(sock.fileno()) + sock = _new_tcp_socket(af, socktype, proto) connected = False try: - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - _set_keepalive_times(sock) - sock.setblocking(False) + _configure_tcp_socket(sock, non_blocking=True) async with asyncio.timeout(_proxy_remaining(deadline)): await loop.sock_connect(sock, sockaddr) connected = True await _async_socks5_handshake(sock, address, proxy, deadline) - post_timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout - sock.settimeout(post_timeout) + sock.settimeout(_proxy_socket_timeout(options)) return sock except asyncio.TimeoutError as exc: sock.close() @@ -831,12 +913,6 @@ async def _async_create_proxy_connection( sock.close() if isinstance(exc, _Socks5Error): raise - if isinstance(exc, socket.timeout): - if connected or (deadline is not None and deadline - time.monotonic() <= 0): - raise _Socks5Timeout("timed out") from exc - err = _Socks5Timeout("timed out") - err.__cause__ = exc - continue err = exc # A connected proxy's handshake failure must not try a different # resolved proxy address; the failure is specific to the endpoint. @@ -867,22 +943,15 @@ def _create_proxy_connection( err: Optional[OSError] = None for af, socktype, proto, _, sockaddr in results: - try: - sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) - except OSError: - sock = socket.socket(af, socktype, proto) - _set_non_inheritable_non_atomic(sock.fileno()) + sock = _new_tcp_socket(af, socktype, proto) connected = False try: - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - _set_keepalive_times(sock) + _configure_tcp_socket(sock) sock.settimeout(_proxy_remaining(deadline)) sock.connect(sockaddr) connected = True _socks5_handshake(sock, address, proxy, deadline) - post_timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout - sock.settimeout(post_timeout) + sock.settimeout(_proxy_socket_timeout(options)) return sock except socket.timeout as exc: sock.close() @@ -949,26 +1018,16 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s # SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited # number of platforms (newer Linux and *BSD). Starting with CPython 3.4 # all file descriptors are created non-inheritable. See PEP 446. + sock = _new_tcp_socket(af, socktype, proto) try: - sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) - except OSError: - # Can SOCK_CLOEXEC be defined even if the kernel doesn't support - # it? - sock = socket.socket(af, socktype, proto) - # Fallback when SOCK_CLOEXEC isn't available. - _set_non_inheritable_non_atomic(sock.fileno()) - try: - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + _configure_tcp_socket(sock, non_blocking=True) # CSOT: apply timeout to socket connect. timeout = _csot.remaining() if timeout is None: timeout = options.connect_timeout elif timeout <= 0: raise socket.timeout("timed out") - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - _set_keepalive_times(sock) # Socket needs to be non-blocking during connection to not block the event loop - sock.setblocking(False) await asyncio.wait_for( asyncio.get_running_loop().sock_connect(sock, sa), timeout=timeout ) @@ -1198,16 +1257,9 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket # SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited # number of platforms (newer Linux and *BSD). Starting with CPython 3.4 # all file descriptors are created non-inheritable. See PEP 446. + sock = _new_tcp_socket(af, socktype, proto) try: - sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) - except OSError: - # Can SOCK_CLOEXEC be defined even if the kernel doesn't support - # it? - sock = socket.socket(af, socktype, proto) - # Fallback when SOCK_CLOEXEC isn't available. - _set_non_inheritable_non_atomic(sock.fileno()) - try: - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + _configure_tcp_socket(sock) # CSOT: apply timeout to socket connect. timeout = _csot.remaining() if timeout is None: @@ -1215,8 +1267,6 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket elif timeout <= 0: raise socket.timeout("timed out") sock.settimeout(timeout) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - _set_keepalive_times(sock) sock.connect(sa) return sock except OSError as e: diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 7988b5a8e7..86abd73ddb 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -93,6 +93,7 @@ UpdateOne, _Op, ) +from pymongo.pool_options import _ProxyOptions from pymongo.read_preferences import ReadPreference, _ServerMode from pymongo.results import ClientBulkWriteResult from pymongo.server_description import ServerDescription @@ -675,6 +676,10 @@ def validator(host: str) -> bool: .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.20 + Added the ``proxyHost``, ``proxyPort``, ``proxyUsername``, and + ``proxyPassword`` keyword arguments and URI options. + .. versionchanged:: 4.19 Added the ``srv_host_validator`` keyword argument. @@ -1126,9 +1131,6 @@ def _validate_kwargs_and_update_opts( keyword_opts = common._CaseInsensitiveDictionary( dict(common.validate(keyword_opts.cased_key(k), v) for k, v in keyword_opts.items()) ) - for name in ("proxyusername", "proxypassword"): - if keyword_opts.get(name) == "": - keyword_opts.pop(name) # Override connection string options with kwarg options. opts.update(keyword_opts) return opts @@ -1387,12 +1389,15 @@ def options(self) -> ClientOptions: """ return self._options - def eq_props(self) -> tuple[tuple[_Address, ...], Optional[str], Optional[str], str]: + def eq_props( + self, + ) -> tuple[tuple[_Address, ...], Optional[str], Optional[str], str, Optional[_ProxyOptions]]: return ( tuple(sorted(self._resolve_srv_info["seeds"])), self._options.replica_set_name, self._resolve_srv_info["fqdn"], self._resolve_srv_info["srv_service_name"], + self._options.pool_options._proxy_options, ) def __eq__(self, other: Any) -> bool: diff --git a/pymongo/uri_parser_shared.py b/pymongo/uri_parser_shared.py index 45c99f1110..baad74874c 100644 --- a/pymongo/uri_parser_shared.py +++ b/pymongo/uri_parser_shared.py @@ -481,6 +481,8 @@ def split_hosts(hosts: str, default_port: Optional[int] = DEFAULT_PORT) -> list[ # backward-compat we allow "db.collection" in URI. _BAD_DB_CHARS = re.compile("[" + re.escape(r'/ "$') + "]") +_SOCKS5_MAX_STR_LEN = 255 + _ALLOWED_TXT_OPTS = frozenset( ["authsource", "authSource", "replicaset", "replicaSet", "loadbalanced", "loadBalanced"] ) @@ -508,8 +510,10 @@ def _check_proxy_options(nodes: Sized, options: Mapping[str, Any]) -> None: if bool(proxy_username) != bool(proxy_password): raise ConfigurationError("Both proxyUsername and proxyPassword must be specified together") for name, value in (("proxyUsername", proxy_username), ("proxyPassword", proxy_password)): - if value is not None and len(value.encode("utf-8")) > 255: - raise ConfigurationError(f"{name} must be no more than 255 UTF-8 bytes") + if value is not None and len(value.encode("utf-8")) > _SOCKS5_MAX_STR_LEN: + raise ConfigurationError( + f"{name} must be no more than {_SOCKS5_MAX_STR_LEN} UTF-8 bytes" + ) if any(host.endswith(".sock") for host, _ in cast(Iterable[_Address], nodes)): raise ConfigurationError("Cannot specify proxyHost with Unix domain sockets") diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index ebcf0f2db6..76e83321da 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -37,144 +37,12 @@ from pymongo.pool_shared import _async_configured_socket, _async_create_connection, _Socks5Error from test.asynchronous import AsyncPyMongoTestCase, unittest from test.helpers_shared import CERT_PATH +from test.socks5_helpers import Socks5Server, _Socks5Server _IS_SYNC = False pytestmark = pytest.mark.default_async -class _Socks5Server: - def __init__( - self, - method: int = 0, - auth_status: int = 0, - reply: int = 0, - bound_atyp: int = 1, - payload: bytes = b"", - method_version: int = 5, - reply_version: int = 5, - eof_stage: Optional[str] = None, - stall: bool = False, - target: Optional[tuple[str, int]] = None, - ) -> None: - self.method = method - self.auth_status = auth_status - self.reply = reply - self.bound_atyp = bound_atyp - self.payload = payload - self.method_version = method_version - self.reply_version = reply_version - self.eof_stage = eof_stage - self.stall = stall - self.target = target - self.started = threading.Event() - self.greeting = b"" - self.auth = b"" - self.request = b"" - self.error: Optional[BaseException] = None - self.listener = socket.socket() - self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - self.listener.bind(("127.0.0.1", 0)) - self.listener.listen(1) - self.listener.settimeout(3) - self.address = self.listener.getsockname() - self.thread = threading.Thread(target=self._serve, daemon=True) - self.thread.start() - - @staticmethod - def _read(sock: socket.socket, size: int) -> bytes: - data = bytearray() - while len(data) < size: - part = sock.recv(size - len(data)) - if not part: - raise EOFError("client closed connection") - data.extend(part) - return bytes(data) - - def _serve(self) -> None: - upstream: Optional[socket.socket] = None - try: - with self.listener.accept()[0] as conn: - conn.settimeout(3) - version, nmethods = self._read(conn, 2) - self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) - self.started.set() - if self.stall: - conn.recv(1) - return - if self.eof_stage == "greeting": - conn.sendall(b"\x05") - return - conn.sendall(bytes((self.method_version, self.method))) - if self.method == 2: - auth_version, username_length = self._read(conn, 2) - username = self._read(conn, username_length) - password_length = self._read(conn, 1)[0] - password = self._read(conn, password_length) - self.auth = ( - bytes((auth_version, username_length)) - + username - + bytes((password_length,)) - + password - ) - if self.eof_stage == "auth": - conn.sendall(b"\x01") - return - conn.sendall(bytes((1, self.auth_status))) - if self.auth_status: - return - version, command, reserved, atyp = self._read(conn, 4) - if atyp == 1: - addr = self._read(conn, 4) - elif atyp == 4: - addr = self._read(conn, 16) - else: - length = self._read(conn, 1) - addr = length + self._read(conn, length[0]) - self.request = ( - bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) - ) - if self.eof_stage == "connect": - conn.sendall(b"\x05\x00") - return - if self.target is not None: - upstream = socket.create_connection(self.target, timeout=3) - if self.bound_atyp == 1: - bound = b"\x01\x7f\x00\x00\x01\x00\x00" - elif self.bound_atyp == 4: - bound = b"\x04" + (b"\x00" * 16) + b"\x00\x00" - else: - bound = b"\x03\x01x\x00\x00" - conn.sendall(bytes((self.reply_version, self.reply, 0)) + bound + self.payload) - if self.payload: - return - if upstream is not None: - conn.setblocking(False) - upstream.setblocking(False) - while True: - readable, _, _ = select.select([conn, upstream], [], [], 5) - if not readable: - return - for source in readable: - data = source.recv(65536) - if not data: - return - destination = upstream if source is conn else conn - destination.sendall(data) - else: - while conn.recv(1024): - pass - except BaseException as exc: - self.error = exc - finally: - if upstream is not None: - upstream.close() - self.listener.close() - - def close(self) -> None: - self.listener.close() - self.thread.join(timeout=3) - - @contextmanager def _refuse_connection(address: tuple[str, int]) -> Iterator[None]: if _IS_SYNC: @@ -319,7 +187,8 @@ async def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(se from pymongo.asynchronous.mongo_client import AsyncMongoClient client = AsyncMongoClient( - "mongodb://localhost/?proxyHost=localhost&proxyPort=1234", + "mongodb://localhost/?proxyHost=localhost&proxyPort=1234" + "&proxyUsername=uri-user&proxyPassword=uri-password", proxyHost="[::1]", proxyPort=5678, proxyUsername="", @@ -331,6 +200,21 @@ async def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(se finally: await client.close() + async def test_client_equality_includes_proxy_options(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + clients = [ + AsyncMongoClient("mongodb://localhost/?proxyHost=proxy.example", connect=False), + AsyncMongoClient("mongodb://localhost/?proxyHost=proxy.example", connect=False), + AsyncMongoClient("mongodb://localhost/?proxyHost=other.example", connect=False), + ] + try: + self.assertEqual(clients[0], clients[1]) + self.assertNotEqual(clients[0], clients[2]) + finally: + for client in clients: + await client.close() + async def test_bad_versions_and_unexpected_eof_fail(self) -> None: cases = ( (_Socks5Server(method_version=4), ("localhost", 27017), None), @@ -355,7 +239,7 @@ async def test_timeout_covers_the_entire_handshake(self) -> None: await self._connect(server, ("localhost", 27017), timeout=0.1) server.close() - @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the asynchronous driver") + @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") async def test_cancellation_closes_proxy_socket(self) -> None: server = _Socks5Server(stall=True) self.addCleanup(server.close) @@ -365,7 +249,7 @@ async def test_cancellation_closes_proxy_socket(self) -> None: with self.assertRaises(asyncio.CancelledError): await task - @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the asynchronous driver") + @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") async def test_cancellation_when_proxy_send_completes_closes_socket(self) -> None: server = _Socks5Server(stall=True) self.addCleanup(server.close) @@ -758,8 +642,6 @@ async def close() -> None: await client.close() async def test_in_process_socks5_server_forwarding(self) -> None: - from test.asynchronous.test_socks5_prose import Socks5Server - echo_listener = socket.socket() echo_listener.bind(("127.0.0.1", 0)) echo_listener.listen(2) diff --git a/test/asynchronous/test_socks5_prose.py b/test/asynchronous/test_socks5_prose.py index d49f8f6fa7..3ff8d23ced 100644 --- a/test/asynchronous/test_socks5_prose.py +++ b/test/asynchronous/test_socks5_prose.py @@ -30,6 +30,7 @@ from pymongo.hello import HelloCompat from test.asynchronous import AsyncIntegrationTest, async_client_context from test.helpers_shared import TLS_OPTIONS +from test.socks5_helpers import Socks5Server _IS_SYNC = False pytestmark = pytest.mark.default_async @@ -49,138 +50,6 @@ def failed(self, event: monitoring.CommandFailedEvent) -> None: self.connection_ids.append(event.connection_id) -class Socks5Server: - """Small threaded SOCKS5 forwarder for the driver prose tests.""" - - def __init__( - self, - port: int, - mapping: dict[tuple[str, int], tuple[str, int]], - credentials: Optional[tuple[str, str]] = None, - ) -> None: - self.mapping = mapping - self.credentials = credentials - self.listener = socket.socket() - self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - try: - self.listener.bind(("127.0.0.1", port)) - except OSError: - self.listener.bind(("127.0.0.1", 0)) - self.port = self.listener.getsockname()[1] - self.listener.listen() - self.listener.settimeout(0.5) - self._closed = threading.Event() - self._threads: list[threading.Thread] = [] - self._thread = threading.Thread(target=self._serve, daemon=True) - self._thread.start() - - @staticmethod - def _read(sock: socket.socket, length: int) -> bytes: - data = bytearray() - while len(data) < length: - part = sock.recv(length - len(data)) - if not part: - raise EOFError - data.extend(part) - return bytes(data) - - def _serve_client(self, client: socket.socket) -> None: - upstream: Optional[socket.socket] = None - try: - client.settimeout(5) - version, method_count = self._read(client, 2) - methods = self._read(client, method_count) - if version != 5: - return - if self.credentials is not None: - method = 2 if 2 in methods else 0xFF - else: - method = 0 if 0 in methods else 0xFF - client.sendall(bytes((5, method))) - if method == 0xFF: - return - if method == 2: - version, username_length = self._read(client, 2) - username = self._read(client, username_length) - password_length = self._read(client, 1)[0] - password = self._read(client, password_length) - user, secret = self.credentials or ("", "") - status = ( - 0 - if version == 1 and username == user.encode() and password == secret.encode() - else 1 - ) - client.sendall(bytes((1, status))) - if status: - return - version, command, reserved, atyp = self._read(client, 4) - if atyp == 1: - host = socket.inet_ntop(socket.AF_INET, self._read(client, 4)) - elif atyp == 4: - host = socket.inet_ntop(socket.AF_INET6, self._read(client, 16)) - elif atyp == 3: - length = self._read(client, 1)[0] - host = self._read(client, length).decode("idna") - else: - client.sendall(b"\x05\x08\x00\x01\x00\x00\x00\x00\x00\x00") - return - port = int.from_bytes(self._read(client, 2), "big") - if version != 5 or command != 1 or reserved != 0: - client.sendall(b"\x05\x07\x00\x01\x00\x00\x00\x00\x00\x00") - return - target = self.mapping.get((host.lower(), port), (host, port)) - try: - upstream = socket.create_connection(target, timeout=5) - except OSError: - client.sendall(b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00") - return - bound_host, bound_port = upstream.getsockname()[:2] - try: - bound_ip = socket.inet_pton(socket.AF_INET, bound_host) - atyp = 1 - except OSError: - bound_ip = socket.inet_pton(socket.AF_INET6, bound_host) - atyp = 4 - client.sendall(bytes((5, 0, 0, atyp)) + bound_ip + int(bound_port).to_bytes(2, "big")) - client.setblocking(False) - upstream.setblocking(False) - while True: - readable, _, _ = select.select([client, upstream], [], [], 10) - if not readable: - continue - for source in readable: - data = source.recv(65536) - if not data: - return - target_sock = upstream if source is client else client - target_sock.sendall(data) - except (OSError, EOFError): - pass - finally: - if upstream is not None: - upstream.close() - client.close() - - def _serve(self) -> None: - while not self._closed.is_set(): - try: - client, _ = self.listener.accept() - except socket.timeout: - continue - except OSError: - break - thread = threading.Thread(target=self._serve_client, args=(client,), daemon=True) - self._threads.append(thread) - thread.start() - - def close(self) -> None: - self._closed.set() - self.listener.close() - self._thread.join(timeout=2) - for thread in self._threads: - thread.join(timeout=2) - - class TestSocks5Prose(AsyncIntegrationTest): @async_client_context.require_replica_set async def asyncSetUp(self) -> None: diff --git a/test/socks5_helpers.py b/test/socks5_helpers.py new file mode 100644 index 0000000000..8b66cba519 --- /dev/null +++ b/test/socks5_helpers.py @@ -0,0 +1,287 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Threaded SOCKS5 test servers shared by synchronous and asynchronous tests.""" + +from __future__ import annotations + +import select +import socket +import threading +from typing import Optional + + +class _Socks5Server: + def __init__( + self, + method: int = 0, + auth_status: int = 0, + reply: int = 0, + bound_atyp: int = 1, + payload: bytes = b"", + method_version: int = 5, + reply_version: int = 5, + eof_stage: Optional[str] = None, + stall: bool = False, + target: Optional[tuple[str, int]] = None, + ) -> None: + self.method = method + self.auth_status = auth_status + self.reply = reply + self.bound_atyp = bound_atyp + self.payload = payload + self.method_version = method_version + self.reply_version = reply_version + self.eof_stage = eof_stage + self.stall = stall + self.target = target + self.started = threading.Event() + self.greeting = b"" + self.auth = b"" + self.request = b"" + self.error: Optional[BaseException] = None + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.listener.settimeout(3) + self.address = self.listener.getsockname() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + @staticmethod + def _read(sock: socket.socket, size: int) -> bytes: + data = bytearray() + while len(data) < size: + part = sock.recv(size - len(data)) + if not part: + raise EOFError("client closed connection") + data.extend(part) + return bytes(data) + + def _serve(self) -> None: + upstream: Optional[socket.socket] = None + try: + with self.listener.accept()[0] as conn: + conn.settimeout(3) + version, nmethods = self._read(conn, 2) + self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) + self.started.set() + if self.stall: + conn.recv(1) + return + if self.eof_stage == "greeting": + conn.sendall(b"\x05") + return + conn.sendall(bytes((self.method_version, self.method))) + if self.method == 2: + auth_version, username_length = self._read(conn, 2) + username = self._read(conn, username_length) + password_length = self._read(conn, 1)[0] + password = self._read(conn, password_length) + self.auth = ( + bytes((auth_version, username_length)) + + username + + bytes((password_length,)) + + password + ) + if self.eof_stage == "auth": + conn.sendall(b"\x01") + return + conn.sendall(bytes((1, self.auth_status))) + if self.auth_status: + return + version, command, reserved, atyp = self._read(conn, 4) + if atyp == 1: + addr = self._read(conn, 4) + elif atyp == 4: + addr = self._read(conn, 16) + else: + length = self._read(conn, 1) + addr = length + self._read(conn, length[0]) + self.request = ( + bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) + ) + if self.eof_stage == "connect": + conn.sendall(b"\x05\x00") + return + if self.target is not None: + upstream = socket.create_connection(self.target, timeout=3) + if self.bound_atyp == 1: + bound = b"\x01\x7f\x00\x00\x01\x00\x00" + elif self.bound_atyp == 4: + bound = b"\x04" + (b"\x00" * 16) + b"\x00\x00" + else: + bound = b"\x03\x01x\x00\x00" + conn.sendall(bytes((self.reply_version, self.reply, 0)) + bound + self.payload) + if self.payload: + return + if upstream is not None: + conn.setblocking(False) + upstream.setblocking(False) + while True: + readable, _, _ = select.select([conn, upstream], [], [], 5) + if not readable: + return + for source in readable: + data = source.recv(65536) + if not data: + return + destination = upstream if source is conn else conn + destination.sendall(data) + else: + while conn.recv(1024): + pass + except BaseException as exc: + self.error = exc + finally: + if upstream is not None: + upstream.close() + self.listener.close() + + def close(self) -> None: + self.listener.close() + self.thread.join(timeout=3) + + +class Socks5Server: + """Small threaded SOCKS5 forwarder for the driver prose tests.""" + + def __init__( + self, + port: int, + mapping: dict[tuple[str, int], tuple[str, int]], + credentials: Optional[tuple[str, str]] = None, + ) -> None: + self.mapping = mapping + self.credentials = credentials + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + try: + self.listener.bind(("127.0.0.1", port)) + except OSError: + self.listener.bind(("127.0.0.1", 0)) + self.port = self.listener.getsockname()[1] + self.listener.listen() + self.listener.settimeout(0.5) + self._closed = threading.Event() + self._threads: list[threading.Thread] = [] + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + @staticmethod + def _read(sock: socket.socket, length: int) -> bytes: + data = bytearray() + while len(data) < length: + part = sock.recv(length - len(data)) + if not part: + raise EOFError + data.extend(part) + return bytes(data) + + def _serve_client(self, client: socket.socket) -> None: + upstream: Optional[socket.socket] = None + try: + client.settimeout(5) + version, method_count = self._read(client, 2) + methods = self._read(client, method_count) + if version != 5: + return + if self.credentials is not None: + method = 2 if 2 in methods else 0xFF + else: + method = 0 if 0 in methods else 0xFF + client.sendall(bytes((5, method))) + if method == 0xFF: + return + if method == 2: + version, username_length = self._read(client, 2) + username = self._read(client, username_length) + password_length = self._read(client, 1)[0] + password = self._read(client, password_length) + user, secret = self.credentials or ("", "") + status = ( + 0 + if version == 1 and username == user.encode() and password == secret.encode() + else 1 + ) + client.sendall(bytes((1, status))) + if status: + return + version, command, reserved, atyp = self._read(client, 4) + if atyp == 1: + host = socket.inet_ntop(socket.AF_INET, self._read(client, 4)) + elif atyp == 4: + host = socket.inet_ntop(socket.AF_INET6, self._read(client, 16)) + elif atyp == 3: + length = self._read(client, 1)[0] + host = self._read(client, length).decode("idna") + else: + client.sendall(b"\x05\x08\x00\x01\x00\x00\x00\x00\x00\x00") + return + port = int.from_bytes(self._read(client, 2), "big") + if version != 5 or command != 1 or reserved != 0: + client.sendall(b"\x05\x07\x00\x01\x00\x00\x00\x00\x00\x00") + return + target = self.mapping.get((host.lower(), port), (host, port)) + try: + upstream = socket.create_connection(target, timeout=5) + except OSError: + client.sendall(b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00") + return + bound_host, bound_port = upstream.getsockname()[:2] + try: + bound_ip = socket.inet_pton(socket.AF_INET, bound_host) + atyp = 1 + except OSError: + bound_ip = socket.inet_pton(socket.AF_INET6, bound_host) + atyp = 4 + client.sendall(bytes((5, 0, 0, atyp)) + bound_ip + int(bound_port).to_bytes(2, "big")) + client.setblocking(False) + upstream.setblocking(False) + while True: + readable, _, _ = select.select([client, upstream], [], [], 10) + if not readable: + continue + for source in readable: + data = source.recv(65536) + if not data: + return + target_sock = upstream if source is client else client + target_sock.sendall(data) + except (OSError, EOFError): + pass + finally: + if upstream is not None: + upstream.close() + client.close() + + def _serve(self) -> None: + while not self._closed.is_set(): + try: + client, _ = self.listener.accept() + except socket.timeout: + continue + except OSError: + break + thread = threading.Thread(target=self._serve_client, args=(client,), daemon=True) + self._threads.append(thread) + thread.start() + + def close(self) -> None: + self._closed.set() + self.listener.close() + self._thread.join(timeout=2) + for thread in self._threads: + thread.join(timeout=2) diff --git a/test/test_socks5.py b/test/test_socks5.py index f3bb952f6a..52b7ec1a8b 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -37,144 +37,12 @@ from pymongo.pool_shared import _configured_socket, _create_connection, _Socks5Error from test import PyMongoTestCase, unittest from test.helpers_shared import CERT_PATH +from test.socks5_helpers import Socks5Server, _Socks5Server _IS_SYNC = True pytestmark = pytest.mark.default -class _Socks5Server: - def __init__( - self, - method: int = 0, - auth_status: int = 0, - reply: int = 0, - bound_atyp: int = 1, - payload: bytes = b"", - method_version: int = 5, - reply_version: int = 5, - eof_stage: Optional[str] = None, - stall: bool = False, - target: Optional[tuple[str, int]] = None, - ) -> None: - self.method = method - self.auth_status = auth_status - self.reply = reply - self.bound_atyp = bound_atyp - self.payload = payload - self.method_version = method_version - self.reply_version = reply_version - self.eof_stage = eof_stage - self.stall = stall - self.target = target - self.started = threading.Event() - self.greeting = b"" - self.auth = b"" - self.request = b"" - self.error: Optional[BaseException] = None - self.listener = socket.socket() - self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - self.listener.bind(("127.0.0.1", 0)) - self.listener.listen(1) - self.listener.settimeout(3) - self.address = self.listener.getsockname() - self.thread = threading.Thread(target=self._serve, daemon=True) - self.thread.start() - - @staticmethod - def _read(sock: socket.socket, size: int) -> bytes: - data = bytearray() - while len(data) < size: - part = sock.recv(size - len(data)) - if not part: - raise EOFError("client closed connection") - data.extend(part) - return bytes(data) - - def _serve(self) -> None: - upstream: Optional[socket.socket] = None - try: - with self.listener.accept()[0] as conn: - conn.settimeout(3) - version, nmethods = self._read(conn, 2) - self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) - self.started.set() - if self.stall: - conn.recv(1) - return - if self.eof_stage == "greeting": - conn.sendall(b"\x05") - return - conn.sendall(bytes((self.method_version, self.method))) - if self.method == 2: - auth_version, username_length = self._read(conn, 2) - username = self._read(conn, username_length) - password_length = self._read(conn, 1)[0] - password = self._read(conn, password_length) - self.auth = ( - bytes((auth_version, username_length)) - + username - + bytes((password_length,)) - + password - ) - if self.eof_stage == "auth": - conn.sendall(b"\x01") - return - conn.sendall(bytes((1, self.auth_status))) - if self.auth_status: - return - version, command, reserved, atyp = self._read(conn, 4) - if atyp == 1: - addr = self._read(conn, 4) - elif atyp == 4: - addr = self._read(conn, 16) - else: - length = self._read(conn, 1) - addr = length + self._read(conn, length[0]) - self.request = ( - bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) - ) - if self.eof_stage == "connect": - conn.sendall(b"\x05\x00") - return - if self.target is not None: - upstream = socket.create_connection(self.target, timeout=3) - if self.bound_atyp == 1: - bound = b"\x01\x7f\x00\x00\x01\x00\x00" - elif self.bound_atyp == 4: - bound = b"\x04" + (b"\x00" * 16) + b"\x00\x00" - else: - bound = b"\x03\x01x\x00\x00" - conn.sendall(bytes((self.reply_version, self.reply, 0)) + bound + self.payload) - if self.payload: - return - if upstream is not None: - conn.setblocking(False) - upstream.setblocking(False) - while True: - readable, _, _ = select.select([conn, upstream], [], [], 5) - if not readable: - return - for source in readable: - data = source.recv(65536) - if not data: - return - destination = upstream if source is conn else conn - destination.sendall(data) - else: - while conn.recv(1024): - pass - except BaseException as exc: - self.error = exc - finally: - if upstream is not None: - upstream.close() - self.listener.close() - - def close(self) -> None: - self.listener.close() - self.thread.join(timeout=3) - - @contextmanager def _refuse_connection(address: tuple[str, int]) -> Iterator[None]: if _IS_SYNC: @@ -319,7 +187,8 @@ def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(self) -> from pymongo.synchronous.mongo_client import MongoClient client = MongoClient( - "mongodb://localhost/?proxyHost=localhost&proxyPort=1234", + "mongodb://localhost/?proxyHost=localhost&proxyPort=1234" + "&proxyUsername=uri-user&proxyPassword=uri-password", proxyHost="[::1]", proxyPort=5678, proxyUsername="", @@ -331,6 +200,21 @@ def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(self) -> finally: client.close() + def test_client_equality_includes_proxy_options(self) -> None: + from pymongo.synchronous.mongo_client import MongoClient + + clients = [ + MongoClient("mongodb://localhost/?proxyHost=proxy.example", connect=False), + MongoClient("mongodb://localhost/?proxyHost=proxy.example", connect=False), + MongoClient("mongodb://localhost/?proxyHost=other.example", connect=False), + ] + try: + self.assertEqual(clients[0], clients[1]) + self.assertNotEqual(clients[0], clients[2]) + finally: + for client in clients: + client.close() + def test_bad_versions_and_unexpected_eof_fail(self) -> None: cases = ( (_Socks5Server(method_version=4), ("localhost", 27017), None), @@ -355,7 +239,7 @@ def test_timeout_covers_the_entire_handshake(self) -> None: self._connect(server, ("localhost", 27017), timeout=0.1) server.close() - @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the synchronous driver") + @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") def test_cancellation_closes_proxy_socket(self) -> None: server = _Socks5Server(stall=True) self.addCleanup(server.close) @@ -365,7 +249,7 @@ def test_cancellation_closes_proxy_socket(self) -> None: with self.assertRaises(asyncio.CancelledError): task - @unittest.skipIf(_IS_SYNC, "Async cancellation is tested in the synchronous driver") + @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") def test_cancellation_when_proxy_send_completes_closes_socket(self) -> None: server = _Socks5Server(stall=True) self.addCleanup(server.close) @@ -758,8 +642,6 @@ def close() -> None: client.close() def test_in_process_socks5_server_forwarding(self) -> None: - from test.test_socks5_prose import Socks5Server - echo_listener = socket.socket() echo_listener.bind(("127.0.0.1", 0)) echo_listener.listen(2) diff --git a/test/test_socks5_prose.py b/test/test_socks5_prose.py index 9e4680d3b5..da88f7f11f 100644 --- a/test/test_socks5_prose.py +++ b/test/test_socks5_prose.py @@ -30,6 +30,7 @@ from pymongo.synchronous.mongo_client import MongoClient from test import IntegrationTest, client_context from test.helpers_shared import TLS_OPTIONS +from test.socks5_helpers import Socks5Server _IS_SYNC = True pytestmark = pytest.mark.default @@ -49,138 +50,6 @@ def failed(self, event: monitoring.CommandFailedEvent) -> None: self.connection_ids.append(event.connection_id) -class Socks5Server: - """Small threaded SOCKS5 forwarder for the driver prose tests.""" - - def __init__( - self, - port: int, - mapping: dict[tuple[str, int], tuple[str, int]], - credentials: Optional[tuple[str, str]] = None, - ) -> None: - self.mapping = mapping - self.credentials = credentials - self.listener = socket.socket() - self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - try: - self.listener.bind(("127.0.0.1", port)) - except OSError: - self.listener.bind(("127.0.0.1", 0)) - self.port = self.listener.getsockname()[1] - self.listener.listen() - self.listener.settimeout(0.5) - self._closed = threading.Event() - self._threads: list[threading.Thread] = [] - self._thread = threading.Thread(target=self._serve, daemon=True) - self._thread.start() - - @staticmethod - def _read(sock: socket.socket, length: int) -> bytes: - data = bytearray() - while len(data) < length: - part = sock.recv(length - len(data)) - if not part: - raise EOFError - data.extend(part) - return bytes(data) - - def _serve_client(self, client: socket.socket) -> None: - upstream: Optional[socket.socket] = None - try: - client.settimeout(5) - version, method_count = self._read(client, 2) - methods = self._read(client, method_count) - if version != 5: - return - if self.credentials is not None: - method = 2 if 2 in methods else 0xFF - else: - method = 0 if 0 in methods else 0xFF - client.sendall(bytes((5, method))) - if method == 0xFF: - return - if method == 2: - version, username_length = self._read(client, 2) - username = self._read(client, username_length) - password_length = self._read(client, 1)[0] - password = self._read(client, password_length) - user, secret = self.credentials or ("", "") - status = ( - 0 - if version == 1 and username == user.encode() and password == secret.encode() - else 1 - ) - client.sendall(bytes((1, status))) - if status: - return - version, command, reserved, atyp = self._read(client, 4) - if atyp == 1: - host = socket.inet_ntop(socket.AF_INET, self._read(client, 4)) - elif atyp == 4: - host = socket.inet_ntop(socket.AF_INET6, self._read(client, 16)) - elif atyp == 3: - length = self._read(client, 1)[0] - host = self._read(client, length).decode("idna") - else: - client.sendall(b"\x05\x08\x00\x01\x00\x00\x00\x00\x00\x00") - return - port = int.from_bytes(self._read(client, 2), "big") - if version != 5 or command != 1 or reserved != 0: - client.sendall(b"\x05\x07\x00\x01\x00\x00\x00\x00\x00\x00") - return - target = self.mapping.get((host.lower(), port), (host, port)) - try: - upstream = socket.create_connection(target, timeout=5) - except OSError: - client.sendall(b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00") - return - bound_host, bound_port = upstream.getsockname()[:2] - try: - bound_ip = socket.inet_pton(socket.AF_INET, bound_host) - atyp = 1 - except OSError: - bound_ip = socket.inet_pton(socket.AF_INET6, bound_host) - atyp = 4 - client.sendall(bytes((5, 0, 0, atyp)) + bound_ip + int(bound_port).to_bytes(2, "big")) - client.setblocking(False) - upstream.setblocking(False) - while True: - readable, _, _ = select.select([client, upstream], [], [], 10) - if not readable: - continue - for source in readable: - data = source.recv(65536) - if not data: - return - target_sock = upstream if source is client else client - target_sock.sendall(data) - except (OSError, EOFError): - pass - finally: - if upstream is not None: - upstream.close() - client.close() - - def _serve(self) -> None: - while not self._closed.is_set(): - try: - client, _ = self.listener.accept() - except socket.timeout: - continue - except OSError: - break - thread = threading.Thread(target=self._serve_client, args=(client,), daemon=True) - self._threads.append(thread) - thread.start() - - def close(self) -> None: - self._closed.set() - self.listener.close() - self._thread.join(timeout=2) - for thread in self._threads: - thread.join(timeout=2) - - class TestSocks5Prose(IntegrationTest): @client_context.require_replica_set def setUp(self) -> None: From b8ff71c91442bdc977d39ad7224be972dfb108e2 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 19:39:14 +0000 Subject: [PATCH 08/16] PYTHON-2943 Address SOCKS5 review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .evergreen/config.yml | 145 ++++++++ .evergreen/scripts/run_socks5_proxy.py | 52 +++ doc/changelog.rst | 4 +- pymongo/asynchronous/encryption.py | 5 +- pymongo/common.py | 1 + pymongo/pool_shared.py | 36 +- pymongo/synchronous/encryption.py | 5 +- pymongo/uri_parser_shared.py | 16 +- test/asynchronous/test_socks5.py | 443 +++++++++++++++---------- test/asynchronous/test_socks5_prose.py | 168 ---------- test/socks5_helpers.py | 132 -------- test/test_socks5.py | 443 +++++++++++++++---------- test/test_socks5_prose.py | 168 ---------- 13 files changed, 754 insertions(+), 864 deletions(-) create mode 100644 .evergreen/scripts/run_socks5_proxy.py delete mode 100644 test/asynchronous/test_socks5_prose.py delete mode 100644 test/test_socks5_prose.py diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 791dd332f0..c7dcfbb49f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -45,6 +45,140 @@ post: - func: "cleanup" tasks: + - name: test-socks5-prose-noauth-nossl + commands: + - func: run server + vars: + TEST_NAME: default + VERSION: "8.0" + TOPOLOGY: replica_set + AUTH: noauth + SSL: nossl + - command: subprocess.exec + type: test + params: + binary: python3 + args: + - .evergreen/scripts/run_socks5_proxy.py + - --port + - "1080" + - --auth + - username:p4ssw0rd + background: true + working_dir: src + include_expansions_in_env: + - MONGODB_URI + - DRIVERS_TOOLS + - command: subprocess.exec + type: test + params: + binary: python3 + args: + - .evergreen/scripts/run_socks5_proxy.py + - --port + - "1081" + background: true + working_dir: src + include_expansions_in_env: + - MONGODB_URI + - DRIVERS_TOOLS + - command: subprocess.exec + type: test + params: + binary: bash + args: + - .evergreen/just.sh + - setup-tests + - default + working_dir: src + include_expansions_in_env: + - AUTH + - SSL + - MONGODB_URI + - UV_PYTHON + - command: subprocess.exec + type: test + params: + binary: bash + args: + - .evergreen/just.sh + - run-tests + - test/test_socks5.py + - test/asynchronous/test_socks5.py + - -k + - TestSocks5Prose + working_dir: src + include_expansions_in_env: + - SOCKS5_PROXIES_STARTED + - MONGODB_URI + + - name: test-socks5-prose-noauth-ssl + commands: + - func: run server + vars: + TEST_NAME: default + VERSION: "8.0" + TOPOLOGY: replica_set + AUTH: noauth + SSL: ssl + - command: subprocess.exec + type: test + params: + binary: python3 + args: + - .evergreen/scripts/run_socks5_proxy.py + - --port + - "1080" + - --auth + - username:p4ssw0rd + background: true + working_dir: src + include_expansions_in_env: + - MONGODB_URI + - DRIVERS_TOOLS + - command: subprocess.exec + type: test + params: + binary: python3 + args: + - .evergreen/scripts/run_socks5_proxy.py + - --port + - "1081" + background: true + working_dir: src + include_expansions_in_env: + - MONGODB_URI + - DRIVERS_TOOLS + - command: subprocess.exec + type: test + params: + binary: bash + args: + - .evergreen/just.sh + - setup-tests + - default + working_dir: src + include_expansions_in_env: + - AUTH + - SSL + - MONGODB_URI + - UV_PYTHON + - command: subprocess.exec + type: test + params: + binary: bash + args: + - .evergreen/just.sh + - run-tests + - test/test_socks5.py + - test/asynchronous/test_socks5.py + - -k + - TestSocks5Prose + working_dir: src + include_expansions_in_env: + - SOCKS5_PROXIES_STARTED + - MONGODB_URI + - name: linkcheck commands: - command: subprocess.exec @@ -82,3 +216,14 @@ buildvariants: patchable: true tasks: - name: resync_specs + + - name: socks5-prose + display_name: "SOCKS5 Prose Tests" + run_on: rhel8.10-small + patchable: true + expansions: + SOCKS5_PROXIES_STARTED: "1" + UV_PYTHON: "3.11" + tasks: + - name: test-socks5-prose-noauth-nossl + - name: test-socks5-prose-noauth-ssl diff --git a/.evergreen/scripts/run_socks5_proxy.py b/.evergreen/scripts/run_socks5_proxy.py new file mode 100644 index 0000000000..98cf45a982 --- /dev/null +++ b/.evergreen/scripts/run_socks5_proxy.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import argparse +import os +import runpy +import sys +from pathlib import Path +from urllib.parse import urlsplit + + +def _first_seed(uri: str) -> str: + authority = urlsplit(uri).netloc.rsplit("@", 1)[-1] + seed = authority.split(",", 1)[0] + if seed.startswith("["): + closing_bracket = seed.find("]") + if closing_bracket < 0: + raise ValueError(f"Invalid IPv6 seed in MONGODB_URI: {seed}") + host = seed[1:closing_bracket] + suffix = seed[closing_bracket + 1 :] + if suffix and not suffix.startswith(":"): + raise ValueError(f"Invalid seed in MONGODB_URI: {seed}") + port = suffix[1:] if suffix else "27017" + else: + host, separator, port = seed.rpartition(":") + if not separator: + host, port = seed, "27017" + if not host or not port.isdecimal(): + raise ValueError(f"Invalid seed in MONGODB_URI: {seed}") + return f"[{host}]:{port}" if ":" in host else f"{host}:{port}" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--auth") + args = parser.parse_args() + + script = Path(os.environ["DRIVERS_TOOLS"]) / ".evergreen" / "socks5srv.py" + sys.argv = [ + str(script), + "--port", + str(args.port), + "--map", + f"localhost:12345 to {_first_seed(os.environ['MONGODB_URI'])}", + ] + if args.auth: + sys.argv.extend(("--auth", args.auth)) + runpy.run_path(str(script), run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/doc/changelog.rst b/doc/changelog.rst index b00b46074b..c4ad313ca5 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,9 +8,7 @@ Changes in Version 4.20.0 (2026/XX/XX) Encryption KMS servers through a SOCKS5 proxy via the ``proxyHost``, ``proxyPort``, ``proxyUsername``, and ``proxyPassword`` URI and keyword options on :class:`~pymongo.synchronous.mongo_client.MongoClient` and - :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient`. - Asynchronous proxy connections support cancellation during connection - establishment and negotiation + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` (`PYTHON-2943`_). - The client handshake metadata now reports a detected coding agent in ``client.env.agent`` (`PYTHON-5929`_). diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 30800109a9..4b6f9c2e0a 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -123,7 +123,7 @@ def __init__( key_vault_coll: AsyncCollection[_DocumentTypeArg], mongocryptd_client: Optional[AsyncMongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, - proxy_client: Optional[AsyncMongoClient[_DocumentTypeArg]] = None, + proxy_client: AsyncMongoClient[_DocumentTypeArg], ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -141,8 +141,7 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client - source_client = proxy_client or client or key_vault_coll.database.client - self._proxy_options = source_client.options.pool_options._proxy_options + self._proxy_options = proxy_client.options.pool_options._proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) diff --git a/pymongo/common.py b/pymongo/common.py index cdf09e083c..9e4ad2368d 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -154,6 +154,7 @@ # URI options that must raise an error instead of warning if they invalidate, # so a malformed proxy option never silently falls back to a direct connection. _URI_OPTS_MUST_RAISE = frozenset(["proxyhost", "proxyport", "proxyusername", "proxypassword"]) +_SOCKS5_MAX_STR_LEN = 255 def partition_node(node: str) -> tuple[str, int]: diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 9306a1ce62..9543b25a3b 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -43,7 +43,13 @@ from pymongo._telemetry import _CmapTelemetry from pymongo.asynchronous.helpers import _getaddrinfo from pymongo.auth_shared import _AuthContext -from pymongo.common import MAX_BSON_SIZE, MAX_MESSAGE_SIZE, MAX_WIRE_VERSION, MAX_WRITE_BATCH_SIZE +from pymongo.common import ( + _SOCKS5_MAX_STR_LEN, + MAX_BSON_SIZE, + MAX_MESSAGE_SIZE, + MAX_WIRE_VERSION, + MAX_WRITE_BATCH_SIZE, +) from pymongo.errors import ( # type:ignore[attr-defined] AutoReconnect, ConnectionFailure, @@ -104,7 +110,6 @@ _SOCKS5_AUTH_REPLY_LEN = 2 _SOCKS5_CONNECT_REPLY_LEN = 4 _SOCKS5_DOMAIN_LENGTH_LEN = 1 -_SOCKS5_MAX_STR_LEN = 255 # Flags recording the counters a checkout has incremented, so a failed # checkout can restore them exactly once (PYTHON-6136). @@ -659,7 +664,9 @@ def _proxy_deadline(options: PoolOptions) -> Optional[float]: def _new_tcp_socket(af: int, socktype: int, proto: int) -> socket.socket: - # SOCK_CLOEXEC is not available on all platforms or kernels. + # SOCK_CLOEXEC was added in CPython 3.2 and is not available on all + # platforms or kernels. PEP 446 makes descriptors non-inheritable by + # default starting in CPython 3.4. try: sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) except OSError: @@ -677,7 +684,10 @@ def _configure_tcp_socket(sock: socket.socket, *, non_blocking: bool = False) -> def _proxy_socket_timeout(options: PoolOptions) -> Optional[float]: - return _csot.remaining() if _csot.get_timeout() else options.connect_timeout + timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout + if timeout is not None and timeout <= 0: + raise _Socks5Timeout("timed out") + return timeout def _socks5_bound_address_size(atyp: int, domain_length: Optional[int] = None) -> int: @@ -750,6 +760,8 @@ def _socks5_check_auth_reply(data: bytes) -> None: def _socks5_request(address: _Address) -> bytes: host, port = address + if port is None: + raise _Socks5Error("SOCKS5 connect failed: destination port is required") try: ip = ipaddress.ip_address(host.split("%", 1)[0]) except ValueError: @@ -764,8 +776,6 @@ def _socks5_request(address: _Address) -> bytes: encoded_host = ( bytes((_SOCKS5_ATYP_IPV4 if ip.version == 4 else _SOCKS5_ATYP_IPV6,)) + ip.packed ) - if port is None: - raise _Socks5Error("SOCKS5 connect failed: destination port is required") return ( bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) + encoded_host @@ -882,7 +892,7 @@ async def _async_create_proxy_connection( results = await _getaddrinfo( proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM ) - except (asyncio.TimeoutError, socket.timeout) as exc: + except TimeoutError as exc: raise _Socks5Timeout("timed out") from exc except OSError as exc: raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc @@ -900,15 +910,12 @@ async def _async_create_proxy_connection( await _async_socks5_handshake(sock, address, proxy, deadline) sock.settimeout(_proxy_socket_timeout(options)) return sock - except asyncio.TimeoutError as exc: + except TimeoutError as exc: sock.close() if connected or (deadline is not None and deadline - time.monotonic() <= 0): raise _Socks5Timeout("timed out") from exc err = _Socks5Timeout("timed out") err.__cause__ = exc - except asyncio.CancelledError: - sock.close() - raise except OSError as exc: sock.close() if isinstance(exc, _Socks5Error): @@ -995,7 +1002,6 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s if not hasattr(socket, "AF_UNIX"): raise ConnectionFailure("UNIX-sockets are not supported on this system") sock = socket.socket(socket.AF_UNIX) - # SOCK_CLOEXEC not supported for Unix sockets. _set_non_inheritable_non_atomic(sock.fileno()) try: sock.setblocking(False) @@ -1015,9 +1021,6 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s err = None for res in await _getaddrinfo(host, port, family=family, type=socket.SOCK_STREAM): af, socktype, proto, dummy, sa = res - # SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited - # number of platforms (newer Linux and *BSD). Starting with CPython 3.4 - # all file descriptors are created non-inheritable. See PEP 446. sock = _new_tcp_socket(af, socktype, proto) try: _configure_tcp_socket(sock, non_blocking=True) @@ -1254,9 +1257,6 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket err = None for res in socket.getaddrinfo(host, port, family=family, type=socket.SOCK_STREAM): # type: ignore[attr-defined, unused-ignore] af, socktype, proto, dummy, sa = res - # SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited - # number of platforms (newer Linux and *BSD). Starting with CPython 3.4 - # all file descriptors are created non-inheritable. See PEP 446. sock = _new_tcp_socket(af, socktype, proto) try: _configure_tcp_socket(sock) diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index caf83f0fec..c2b41882d3 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -122,7 +122,7 @@ def __init__( key_vault_coll: Collection[_DocumentTypeArg], mongocryptd_client: Optional[MongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, - proxy_client: Optional[MongoClient[_DocumentTypeArg]] = None, + proxy_client: MongoClient[_DocumentTypeArg], ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -140,8 +140,7 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client - source_client = proxy_client or client or key_vault_coll.database.client - self._proxy_options = source_client.options.pool_options._proxy_options + self._proxy_options = proxy_client.options.pool_options._proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) diff --git a/pymongo/uri_parser_shared.py b/pymongo/uri_parser_shared.py index baad74874c..745b4c6041 100644 --- a/pymongo/uri_parser_shared.py +++ b/pymongo/uri_parser_shared.py @@ -23,7 +23,7 @@ import re import sys import warnings -from collections.abc import Iterable, Mapping, MutableMapping, Sized +from collections.abc import Collection, Iterable, Mapping, MutableMapping from typing import ( TYPE_CHECKING, Any, @@ -35,6 +35,8 @@ from pymongo.client_options import _parse_ssl_options from pymongo.common import ( + _SOCKS5_MAX_STR_LEN, + _URI_OPTS_MUST_RAISE, INTERNAL_URI_OPTION_NAME_MAP, URI_OPTIONS_DEPRECATION_MAP, _CaseInsensitiveDictionary, @@ -250,7 +252,7 @@ def _parse_options(opts: str, delim: Optional[str]) -> _CaseInsensitiveDictionar options.setdefault(key, []).append(value) else: if key in options: - if key.lower() in ("proxyhost", "proxyport", "proxyusername", "proxypassword"): + if key.lower() in _URI_OPTS_MUST_RAISE: raise InvalidURI(f"Duplicate URI option '{key}' is not allowed") warnings.warn(f"Duplicate URI option '{key}'.", stacklevel=2) if key.lower() == "authmechanismproperties": @@ -481,14 +483,12 @@ def split_hosts(hosts: str, default_port: Optional[int] = DEFAULT_PORT) -> list[ # backward-compat we allow "db.collection" in URI. _BAD_DB_CHARS = re.compile("[" + re.escape(r'/ "$') + "]") -_SOCKS5_MAX_STR_LEN = 255 - _ALLOWED_TXT_OPTS = frozenset( ["authsource", "authSource", "replicaset", "replicaSet", "loadbalanced", "loadBalanced"] ) -def _check_proxy_options(nodes: Sized, options: Mapping[str, Any]) -> None: +def _check_proxy_options(nodes: Iterable[_Address], options: Mapping[str, Any]) -> None: proxy_host = options.get("proxyhost") proxy_port = options.get("proxyport") proxy_username = options.get("proxyusername") @@ -514,11 +514,13 @@ def _check_proxy_options(nodes: Sized, options: Mapping[str, Any]) -> None: raise ConfigurationError( f"{name} must be no more than {_SOCKS5_MAX_STR_LEN} UTF-8 bytes" ) - if any(host.endswith(".sock") for host, _ in cast(Iterable[_Address], nodes)): + if any(host.endswith(".sock") for host, _ in nodes): raise ConfigurationError("Cannot specify proxyHost with Unix domain sockets") -def _check_options(nodes: Sized, options: Mapping[str, Any], validate_proxy: bool = True) -> None: +def _check_options( + nodes: Collection[_Address], options: Mapping[str, Any], validate_proxy: bool = True +) -> None: # Ensure directConnection was not True if there are multiple seeds. if len(nodes) > 1 and options.get("directconnection"): raise ConfigurationError("Cannot specify multiple hosts with directConnection=true") diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index 76e83321da..129ddefc66 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -17,7 +17,7 @@ from __future__ import annotations import asyncio -import select +import os import socket import ssl import sys @@ -27,17 +27,26 @@ from types import SimpleNamespace from typing import Any, Optional, cast from unittest import mock +from urllib.parse import quote_plus import pytest sys.path[0:0] = [""] +from pymongo import monitoring from pymongo.encryption_options import _HAVE_PYMONGOCRYPT +from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.pool_shared import _async_configured_socket, _async_create_connection, _Socks5Error -from test.asynchronous import AsyncPyMongoTestCase, unittest -from test.helpers_shared import CERT_PATH -from test.socks5_helpers import Socks5Server, _Socks5Server +from test.asynchronous import ( + AsyncIntegrationTest, + AsyncPyMongoTestCase, + async_client_context, + unittest, +) +from test.asynchronous.utils import async_wait_until +from test.helpers_shared import CERT_PATH, TLS_OPTIONS +from test.socks5_helpers import _Socks5Server _IS_SYNC = False pytestmark = pytest.mark.default_async @@ -69,6 +78,11 @@ async def sock_connect(sock: socket.socket, target: tuple[str, int]) -> None: class TestSocks5(AsyncPyMongoTestCase): + def _create_server(self, **kwargs: Any) -> _Socks5Server: + server = _Socks5Server(**kwargs) + self.addAsyncCleanup(server.close) + return server + async def _connect( self, server: _Socks5Server, @@ -77,7 +91,7 @@ async def _connect( password: Optional[str] = None, timeout: float = 2, ) -> socket.socket: - return await _async_create_connection( + sock = await _async_create_connection( address, PoolOptions( connect_timeout=timeout, @@ -86,9 +100,11 @@ async def _connect( ), ), ) + self.addAsyncCleanup(sock.close) + return sock async def test_greeting_and_authenticated_connect(self) -> None: - server = _Socks5Server(method=2) + server = self._create_server(method=2) sock = await self._connect(server, ("mongodb.example", 27017), "üser", "påss") sock.close() server.close() @@ -101,7 +117,7 @@ async def test_greeting_and_authenticated_connect(self) -> None: ) async def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: - server = _Socks5Server(method=0) + server = self._create_server(method=0) sock = await self._connect(server, ("127.0.0.1", 27017), "user", "password") sock.close() server.close() @@ -110,7 +126,7 @@ async def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: self.assertEqual(server.request, b"\x05\x01\x00\x01\x7f\x00\x00\x01\x69\x89") async def test_no_credentials_only_offer_no_authentication(self) -> None: - server = _Socks5Server(method=2) + server = self._create_server(method=2) with self.assertRaises(_Socks5Error): await self._connect(server, ("localhost", 27017)) server.close() @@ -118,13 +134,13 @@ async def test_no_credentials_only_offer_no_authentication(self) -> None: async def test_rejects_all_unacceptable_methods(self) -> None: for method in (0x01, 0xFF): - server = _Socks5Server(method=method) + server = self._create_server(method=method) with self.subTest(method=method), self.assertRaises(_Socks5Error): await self._connect(server, ("localhost", 27017)) server.close() async def test_rejects_authentication_failure(self) -> None: - server = _Socks5Server(method=2, auth_status=1) + server = self._create_server(method=2, auth_status=1) with self.assertRaises(_Socks5Error): await self._connect(server, ("localhost", 27017), "user", "password") server.close() @@ -142,7 +158,7 @@ async def test_connect_request_address_types(self) -> None: ), ) for host, expected in cases: - server = _Socks5Server() + server = self._create_server() sock = await self._connect(server, (host, 27017)) sock.close() server.close() @@ -150,7 +166,7 @@ async def test_connect_request_address_types(self) -> None: async def test_drains_bound_address_and_preserves_early_payload(self) -> None: for atyp in (1, 3, 4): - server = _Socks5Server(bound_atyp=atyp, payload=b"mongo") + server = self._create_server(bound_atyp=atyp, payload=b"mongo") sock = await self._connect(server, ("localhost", 27017)) if _IS_SYNC: payload = sock.recv(5) @@ -163,7 +179,7 @@ async def test_drains_bound_address_and_preserves_early_payload(self) -> None: async def test_rejects_socks_reply_codes(self) -> None: for reply in range(1, 9): - server = _Socks5Server(reply=reply) + server = self._create_server(reply=reply) with self.subTest(reply=reply), self.assertRaises(_Socks5Error): await self._connect(server, ("localhost", 27017)) server.close() @@ -217,15 +233,15 @@ async def test_client_equality_includes_proxy_options(self) -> None: async def test_bad_versions_and_unexpected_eof_fail(self) -> None: cases = ( - (_Socks5Server(method_version=4), ("localhost", 27017), None), - (_Socks5Server(reply_version=4), ("localhost", 27017), None), - (_Socks5Server(eof_stage="greeting"), ("localhost", 27017), None), + (self._create_server(method_version=4), ("localhost", 27017), None), + (self._create_server(reply_version=4), ("localhost", 27017), None), + (self._create_server(eof_stage="greeting"), ("localhost", 27017), None), ( - _Socks5Server(method=2, eof_stage="auth"), + self._create_server(method=2, eof_stage="auth"), ("localhost", 27017), ("user", "password"), ), - (_Socks5Server(eof_stage="connect"), ("localhost", 27017), None), + (self._create_server(eof_stage="connect"), ("localhost", 27017), None), ) for server, address, credentials in cases: with self.subTest(stage=server.eof_stage, version=server.method_version): @@ -234,25 +250,23 @@ async def test_bad_versions_and_unexpected_eof_fail(self) -> None: server.close() async def test_timeout_covers_the_entire_handshake(self) -> None: - server = _Socks5Server(stall=True) + server = self._create_server(stall=True) with self.assertRaises(socket.timeout): await self._connect(server, ("localhost", 27017), timeout=0.1) server.close() @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") async def test_cancellation_closes_proxy_socket(self) -> None: - server = _Socks5Server(stall=True) - self.addCleanup(server.close) + server = self._create_server(stall=True) task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) - self.assertTrue(await asyncio.to_thread(server.started.wait, 2)) + await async_wait_until(server.started.is_set, "receive the SOCKS5 greeting", timeout=2) task.cancel() with self.assertRaises(asyncio.CancelledError): await task @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") async def test_cancellation_when_proxy_send_completes_closes_socket(self) -> None: - server = _Socks5Server(stall=True) - self.addCleanup(server.close) + server = self._create_server(stall=True) loop = asyncio.get_running_loop() original_sendall = loop.sock_sendall sockets: list[socket.socket] = [] @@ -269,16 +283,57 @@ async def cancel_after_send(sock: socket.socket, data: bytes) -> None: self.assertEqual(len(sockets), 1) self.assertEqual(sockets[0].fileno(), -1) + @async_client_context.require_async # type: ignore[untyped-decorator] async def test_tls_sni_targets_destination_not_proxy(self) -> None: - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - target_address = listener.getsockname() tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") sni_names: list[Optional[str]] = [] tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + async def serve_tls(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + await reader.read(1) + finally: + writer.close() + await writer.wait_closed() + + tls_server: Any = await asyncio.start_server(serve_tls, "127.0.0.1", 0, ssl=tls_context) + + async def close_tls_server() -> None: + tls_server.close() + await tls_server.wait_closed() + + self.addAsyncCleanup(close_tls_server) + tls_sockets = tls_server.sockets + assert tls_sockets is not None + target_address = tls_sockets[0].getsockname() + proxy = self._create_server(target=target_address) + client_context = ssl.create_default_context() + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + sock = await _async_configured_socket( + ("mongodb.example", target_address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, client_context), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), + ), + ) + self.addAsyncCleanup(sock.close) + sock.close() + self.assertEqual(sni_names, ["mongodb.example"]) + + @async_client_context.require_sync # type: ignore[untyped-decorator] + async def test_tls_sni_targets_destination_not_proxy_sync(self) -> None: + tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") + sni_names: list[Optional[str]] = [] + tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + target_address = listener.getsockname() + def serve_tls() -> None: try: raw, _ = listener.accept() @@ -291,7 +346,13 @@ def serve_tls() -> None: tls_thread = threading.Thread(target=serve_tls, daemon=True) tls_thread.start() - proxy = _Socks5Server(target=target_address) + + def close_tls_server() -> None: + listener.close() + tls_thread.join(timeout=3) + + self.addAsyncCleanup(close_tls_server) + proxy = self._create_server(target=target_address) client_context = ssl.create_default_context() client_context.check_hostname = False client_context.verify_mode = ssl.CERT_NONE @@ -303,9 +364,8 @@ def serve_tls() -> None: proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), ), ) + self.addAsyncCleanup(sock.close) sock.close() - proxy.close() - tls_thread.join(timeout=3) self.assertEqual(sni_names, ["mongodb.example"]) @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") @@ -323,7 +383,13 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: client = AsyncMongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) expected = _ProxyOptions("127.0.0.1") - io = _EncryptionIO(client, client.keyvault.datakeys, None, FakeOpts()) # type: ignore[arg-type] + io = _EncryptionIO( + client, + client.keyvault.datakeys, + None, + FakeOpts(), # type: ignore[arg-type] + proxy_client=client, + ) connection = mock.MagicMock() async def connect(_address, options, _callback, _timeout): @@ -444,8 +510,7 @@ async def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: self.assertFalse(wrapped.has_error_label("RetryableError")) async def test_multi_address_proxy_fallback_on_connect_error(self) -> None: - server = _Socks5Server() - self.addCleanup(server.close) + server = self._create_server() dead_addr = ("127.0.0.1", 1) fake_results = [ @@ -502,15 +567,8 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: await parent_client.close() await kv_client.close() - async def test_socks5_helper_edge_cases(self) -> None: - from pymongo.common import validate_proxy_host, validate_proxy_port - from pymongo.pool_shared import ( - _proxy_deadline, - _proxy_remaining, - _socks5_check_connect_reply, - _socks5_request, - _Socks5Timeout, - ) + async def test_proxy_host_validation(self) -> None: + from pymongo.common import validate_proxy_host self.assertIsNone(validate_proxy_host("proxyHost", None)) self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") @@ -518,10 +576,17 @@ async def test_socks5_helper_edge_cases(self) -> None: validate_proxy_host("proxyHost", "[not-ipv6]") with self.assertRaises(ValueError): validate_proxy_host("proxyHost", "bad/host") + + async def test_proxy_port_validation(self) -> None: + from pymongo.common import validate_proxy_port + self.assertIsNone(validate_proxy_port("proxyPort", None)) with self.assertRaises(TypeError): validate_proxy_port("proxyPort", True) + async def test_proxy_deadline(self) -> None: + from pymongo.pool_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout + self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) self.assertIsNone(_proxy_remaining(None)) with self.assertRaises(_Socks5Timeout): @@ -532,14 +597,43 @@ async def test_socks5_helper_edge_cases(self) -> None: ): _proxy_deadline(PoolOptions(connect_timeout=1)) + async def test_proxy_socket_timeout_rejects_expired_deadline(self) -> None: + from pymongo.pool_shared import _proxy_socket_timeout, _Socks5Timeout + + with ( + mock.patch("pymongo.pool_shared._csot.get_timeout", return_value=True), + mock.patch("pymongo.pool_shared._csot.remaining", return_value=-0.1), + self.assertRaises(_Socks5Timeout), + ): + _proxy_socket_timeout(PoolOptions(connect_timeout=1)) + + async def test_socks5_request_rejects_long_domain(self) -> None: + from pymongo.pool_shared import _socks5_request + with self.assertRaisesRegex(_Socks5Error, "invalid domain name length"): _socks5_request(("a" * 256, 27017)) + + async def test_socks5_request_rejects_invalid_domain(self) -> None: + from pymongo.pool_shared import _socks5_request + with self.assertRaisesRegex(_Socks5Error, "invalid domain name"): _socks5_request((".münich.example", 27017)) + + async def test_socks5_request_checks_port_before_encoding_domain(self) -> None: + from pymongo.pool_shared import _socks5_request + with self.assertRaisesRegex(_Socks5Error, "destination port is required"): - _socks5_request(("localhost", None)) # type: ignore[arg-type] + _socks5_request(("a" * 256, None)) # type: ignore[arg-type] + + async def test_socks5_connect_reply_rejects_invalid_reserved_byte(self) -> None: + from pymongo.pool_shared import _socks5_check_connect_reply + with self.assertRaisesRegex(_Socks5Error, "invalid reserved byte"): _socks5_check_connect_reply(b"\x05\x00\x01\x01") + + async def test_socks5_connect_reply_rejects_invalid_address_type(self) -> None: + from pymongo.pool_shared import _socks5_check_connect_reply + with self.assertRaisesRegex(_Socks5Error, "invalid address type"): _socks5_check_connect_reply(b"\x05\x00\x00\x09") @@ -586,149 +680,136 @@ async def dead_async(*_a: Any, **_kw: Any) -> Any: ): await _async_create_connection(("localhost", 27017), proxy_opts) - async def test_prose_client_options(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - from pymongo.errors import OperationFailure - from test.asynchronous import test_socks5_prose - async def command(name: str) -> None: - if name == "nonexistent_socks5_test_command": - raise OperationFailure("unsupported command") +class _CommandListener(monitoring.CommandListener): + def __init__(self) -> None: + self.connection_ids: list[tuple[str, Optional[int]]] = [] - async def close() -> None: - pass + def started(self, event: monitoring.CommandStartedEvent) -> None: + self.connection_ids.append(event.connection_id) - fake_client = SimpleNamespace(admin=SimpleNamespace(command=command), close=close) - case = test_socks5_prose.TestSocks5Prose() - case.credentials = {"username": "dbuser", "password": "dbpass", "authSource": "admin"} - listener = test_socks5_prose._CommandListener() - for tls in (False, True): - for use_uri in (False, True): - for event_listener in (None, listener): - with self.subTest(tls=tls, use_uri=use_uri, listener=event_listener): - with ( - mock.patch.object(test_socks5_prose.async_client_context, "tls", tls), - mock.patch.object( - test_socks5_prose, "AsyncMongoClient", return_value=fake_client - ) as factory, - ): - await case._run_case( - "localhost:12345", - 1080, - ("proxyuser", "proxypass"), - True, - use_uri, - event_listener, - ) - factory.assert_called_once() - args, kwargs = factory.call_args - client = AsyncMongoClient(*args, connect=False, **kwargs) - try: - self.assertEqual( - client.options.pool_options._proxy_options, - _ProxyOptions("localhost", 1080, "proxyuser", "proxypass"), - ) - self.assertEqual(client.options.direct_connection, True) - self.assertEqual(client.options.server_selection_timeout, 2.5) - self.assertEqual( - client.options.pool_options._ssl_context is not None, tls - ) - self.assertEqual(kwargs["authSource"], "admin") - if event_listener is None: - self.assertNotIn("event_listeners", kwargs) - else: - self.assertEqual(kwargs["event_listeners"], [event_listener]) - finally: - await client.close() - - async def test_in_process_socks5_server_forwarding(self) -> None: - echo_listener = socket.socket() - echo_listener.bind(("127.0.0.1", 0)) - echo_listener.listen(2) - echo_addr = echo_listener.getsockname() - - def serve_echo() -> None: - for _ in range(2): - try: - conn, _ = echo_listener.accept() - except OSError: - return - with conn: - data = conn.recv(64) - if data: - conn.sendall(b"ack:" + data) - - echo_thread = threading.Thread(target=serve_echo, daemon=True) - echo_thread.start() - self.addCleanup(echo_thread.join, 2) - self.addCleanup(echo_listener.close) - - mapping = {("mapped.example", 12345): echo_addr} - auth_proxy = Socks5Server(0, mapping, ("user", "secret")) - noauth_proxy = Socks5Server(0, mapping) - self.addCleanup(auth_proxy.close) - self.addCleanup(noauth_proxy.close) - - sock1 = await _async_create_connection( - ("mapped.example", 12345), - PoolOptions( - connect_timeout=2, - proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "secret"), - ), - ) - try: - if _IS_SYNC: - sock1.settimeout(2) - sock1.sendall(b"hello") - response = sock1.recv(64) - else: - sock1.setblocking(False) - loop = asyncio.get_running_loop() - await loop.sock_sendall(sock1, b"hello") - response = await asyncio.wait_for(loop.sock_recv(sock1, 64), timeout=2) - self.assertEqual(response, b"ack:hello") - finally: - sock1.close() + def succeeded(self, event: monitoring.CommandSucceededEvent) -> None: + self.connection_ids.append(event.connection_id) - sock2 = await _async_create_connection( - echo_addr, - PoolOptions( - connect_timeout=2, - proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), - ), + def failed(self, event: monitoring.CommandFailedEvent) -> None: + self.connection_ids.append(event.connection_id) + + +class TestSocks5Prose(AsyncIntegrationTest): + @async_client_context.require_replica_set + async def asyncSetUp(self) -> None: + if not os.environ.get("SOCKS5_PROXIES_STARTED"): + self.skipTest("SOCKS5 proxies must be started by the Evergreen SOCKS5 tasks.") + await super().asyncSetUp() + self.nodes = sorted(async_client_context.nodes) + self.credentials = getattr(self, "credentials", {}) + self.auth_port = 1080 + self.noauth_port = 1081 + + def _hosts(self, mapped: bool) -> str: + if mapped: + return "localhost:12345" + return ",".join( + f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" + for host, port in self.nodes ) + + async def _run_case( + self, + hosts: str, + port: int, + credentials: Optional[tuple[str, str]], + succeeds: bool, + use_uri: bool, + listener: Optional[_CommandListener] = None, + ) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + + proxy_options = {"proxyHost": "localhost", "proxyPort": port} + if credentials: + proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials + tls_options = dict(TLS_OPTIONS) if async_client_context.tls else {} + options = dict(tls_options) + options.update(self.credentials) + options["serverSelectionTimeoutMS"] = 2500 + if hosts == "localhost:12345": + options["directConnection"] = True + if listener is not None: + options["event_listeners"] = [listener] + if use_uri: + uri_options = "&".join( + f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() + ) + hosts = f"mongodb://{hosts}/?{uri_options}" + else: + options.update(proxy_options) + client = AsyncMongoClient(hosts, **options) try: - if _IS_SYNC: - sock2.settimeout(2) - sock2.sendall(b"world") - response = sock2.recv(64) + if succeeds: + await client.admin.command(HelloCompat.CMD) + if listener is not None: + from pymongo.errors import OperationFailure + + with self.assertRaises(OperationFailure): + await client.admin.command("nonexistent_socks5_test_command") else: - sock2.setblocking(False) - loop = asyncio.get_running_loop() - await loop.sock_sendall(sock2, b"world") - response = await asyncio.wait_for(loop.sock_recv(sock2, 64), timeout=2) - self.assertEqual(response, b"ack:world") + from pymongo.errors import ConnectionFailure + + with self.assertRaises(ConnectionFailure): + await client.admin.command(HelloCompat.CMD) finally: - sock2.close() + await client.close() - with self.assertRaises(_Socks5Error): - await _async_create_connection( - ("mapped.example", 12345), - PoolOptions( - connect_timeout=2, - proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "wrong"), - ), - ) - with ( - mock.patch( - "socket.create_connection", side_effect=ConnectionRefusedError("connection refused") - ), - self.assertRaisesRegex(_Socks5Error, "connection refused"), - ): - await _async_create_connection( - ("127.0.0.1", 1), - PoolOptions( - connect_timeout=2, - proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), - ), - ) + async def test_connection_string_prose_table(self) -> None: + cases = ( + (True, self.auth_port, None, False), + (True, self.noauth_port, None, True), + (False, self.auth_port, None, False), + (False, self.noauth_port, None, True), + (True, self.auth_port, ("nonexistentuser", "badauth"), False), + (True, self.noauth_port, ("nonexistentuser", "badauth"), True), + (False, self.noauth_port, ("nonexistentuser", "badauth"), True), + (True, self.auth_port, ("username", "p4ssw0rd"), True), + (True, self.noauth_port, None, True), + (False, self.auth_port, ("username", "p4ssw0rd"), True), + (False, self.noauth_port, None, True), + ) + mapped_listener = _CommandListener() + rs_listener = _CommandListener() + for index, (mapped, proxy_port, credentials, succeeds) in enumerate(cases): + for use_uri in (True, False): + listener = None + if not use_uri: + if index == 1: + listener = mapped_listener + elif index == 3: + listener = rs_listener + with self.subTest( + case=index, + mapped=mapped, + proxy_port=proxy_port, + credentials=credentials, + use_uri=use_uri, + ): + await self._run_case( + self._hosts(mapped), + proxy_port, + credentials, + succeeds, + use_uri, + listener, + ) + proxy_addresses = { + ("localhost", self.auth_port), + ("localhost", self.noauth_port), + ("127.0.0.1", self.auth_port), + ("127.0.0.1", self.noauth_port), + } + self.assertGreaterEqual(len(mapped_listener.connection_ids), 4) + for connection_id in mapped_listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + self.assertEqual(connection_id, ("localhost", 12345)) + self.assertGreaterEqual(len(rs_listener.connection_ids), 4) + for connection_id in rs_listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + self.assertIn(connection_id, self.nodes) diff --git a/test/asynchronous/test_socks5_prose.py b/test/asynchronous/test_socks5_prose.py deleted file mode 100644 index 3ff8d23ced..0000000000 --- a/test/asynchronous/test_socks5_prose.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2026-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""SOCKS5 connection-string prose tests.""" - -from __future__ import annotations - -import os -import select -import socket -import threading -from typing import Optional -from urllib.parse import quote_plus - -import pytest - -from pymongo import monitoring -from pymongo.asynchronous.mongo_client import AsyncMongoClient -from pymongo.hello import HelloCompat -from test.asynchronous import AsyncIntegrationTest, async_client_context -from test.helpers_shared import TLS_OPTIONS -from test.socks5_helpers import Socks5Server - -_IS_SYNC = False -pytestmark = pytest.mark.default_async - - -class _CommandListener(monitoring.CommandListener): - def __init__(self) -> None: - self.connection_ids: list[tuple[str, Optional[int]]] = [] - - def started(self, event: monitoring.CommandStartedEvent) -> None: - self.connection_ids.append(event.connection_id) - - def succeeded(self, event: monitoring.CommandSucceededEvent) -> None: - self.connection_ids.append(event.connection_id) - - def failed(self, event: monitoring.CommandFailedEvent) -> None: - self.connection_ids.append(event.connection_id) - - -class TestSocks5Prose(AsyncIntegrationTest): - @async_client_context.require_replica_set - async def asyncSetUp(self) -> None: - await super().asyncSetUp() - self.nodes = sorted(async_client_context.nodes) - self.mapping = {("localhost", 12345): self.nodes[0]} - self.credentials = getattr(self, "credentials", {}) - if os.environ.get("SOCKS5_PROXIES_STARTED"): - self.auth_port = 1080 - self.noauth_port = 1081 - else: - auth_proxy = Socks5Server(1080, self.mapping, ("username", "p4ssw0rd")) - noauth_proxy = Socks5Server(1081, self.mapping) - self.addCleanup(auth_proxy.close) - self.addCleanup(noauth_proxy.close) - self.auth_port = auth_proxy.port - self.noauth_port = noauth_proxy.port - - def _hosts(self, mapped: bool) -> str: - if mapped: - return "localhost:12345" - return ",".join( - f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" - for host, port in self.nodes - ) - - async def _run_case( - self, - hosts: str, - port: int, - credentials: Optional[tuple[str, str]], - succeeds: bool, - use_uri: bool, - listener: Optional[_CommandListener] = None, - ) -> None: - proxy_options = {"proxyHost": "localhost", "proxyPort": port} - if credentials: - proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials - tls_options = dict(TLS_OPTIONS) if async_client_context.tls else {} - options = dict(tls_options) - options.update(self.credentials) - options["serverSelectionTimeoutMS"] = 2500 - if hosts == "localhost:12345": - options["directConnection"] = True - if listener is not None: - options["event_listeners"] = [listener] - if use_uri: - uri_options = "&".join( - f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() - ) - hosts = f"mongodb://{hosts}/?{uri_options}" - else: - options.update(proxy_options) - client = AsyncMongoClient(hosts, **options) - try: - if succeeds: - await client.admin.command(HelloCompat.CMD) - if listener is not None: - from pymongo.errors import OperationFailure - - with self.assertRaises(OperationFailure): - await client.admin.command("nonexistent_socks5_test_command") - else: - from pymongo.errors import ConnectionFailure - - with self.assertRaises(ConnectionFailure): - await client.admin.command(HelloCompat.CMD) - finally: - await client.close() - - async def test_connection_string_prose_table(self) -> None: - cases = ( - (True, self.auth_port, None, False), - (True, self.noauth_port, None, True), - (False, self.auth_port, None, False), - (False, self.noauth_port, None, True), - (True, self.auth_port, ("nonexistentuser", "badauth"), False), - (True, self.noauth_port, ("nonexistentuser", "badauth"), True), - (False, self.noauth_port, ("nonexistentuser", "badauth"), True), - (True, self.auth_port, ("username", "p4ssw0rd"), True), - (True, self.noauth_port, None, True), - (False, self.auth_port, ("username", "p4ssw0rd"), True), - (False, self.noauth_port, None, True), - ) - mapped_listener = _CommandListener() - rs_listener = _CommandListener() - for index, (mapped, proxy_port, credentials, succeeds) in enumerate(cases): - for use_uri in (True, False): - listener = None - if not use_uri: - if index == 1: - listener = mapped_listener - elif index == 3: - listener = rs_listener - await self._run_case( - self._hosts(mapped), - proxy_port, - credentials, - succeeds, - use_uri, - listener, - ) - proxy_addresses = { - ("localhost", self.auth_port), - ("localhost", self.noauth_port), - ("127.0.0.1", self.auth_port), - ("127.0.0.1", self.noauth_port), - } - self.assertGreaterEqual(len(mapped_listener.connection_ids), 4) - for connection_id in mapped_listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - self.assertEqual(connection_id, ("localhost", 12345)) - self.assertGreaterEqual(len(rs_listener.connection_ids), 4) - for connection_id in rs_listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - self.assertIn(connection_id, self.nodes) diff --git a/test/socks5_helpers.py b/test/socks5_helpers.py index 8b66cba519..34eefe766d 100644 --- a/test/socks5_helpers.py +++ b/test/socks5_helpers.py @@ -153,135 +153,3 @@ def _serve(self) -> None: def close(self) -> None: self.listener.close() self.thread.join(timeout=3) - - -class Socks5Server: - """Small threaded SOCKS5 forwarder for the driver prose tests.""" - - def __init__( - self, - port: int, - mapping: dict[tuple[str, int], tuple[str, int]], - credentials: Optional[tuple[str, str]] = None, - ) -> None: - self.mapping = mapping - self.credentials = credentials - self.listener = socket.socket() - self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - try: - self.listener.bind(("127.0.0.1", port)) - except OSError: - self.listener.bind(("127.0.0.1", 0)) - self.port = self.listener.getsockname()[1] - self.listener.listen() - self.listener.settimeout(0.5) - self._closed = threading.Event() - self._threads: list[threading.Thread] = [] - self._thread = threading.Thread(target=self._serve, daemon=True) - self._thread.start() - - @staticmethod - def _read(sock: socket.socket, length: int) -> bytes: - data = bytearray() - while len(data) < length: - part = sock.recv(length - len(data)) - if not part: - raise EOFError - data.extend(part) - return bytes(data) - - def _serve_client(self, client: socket.socket) -> None: - upstream: Optional[socket.socket] = None - try: - client.settimeout(5) - version, method_count = self._read(client, 2) - methods = self._read(client, method_count) - if version != 5: - return - if self.credentials is not None: - method = 2 if 2 in methods else 0xFF - else: - method = 0 if 0 in methods else 0xFF - client.sendall(bytes((5, method))) - if method == 0xFF: - return - if method == 2: - version, username_length = self._read(client, 2) - username = self._read(client, username_length) - password_length = self._read(client, 1)[0] - password = self._read(client, password_length) - user, secret = self.credentials or ("", "") - status = ( - 0 - if version == 1 and username == user.encode() and password == secret.encode() - else 1 - ) - client.sendall(bytes((1, status))) - if status: - return - version, command, reserved, atyp = self._read(client, 4) - if atyp == 1: - host = socket.inet_ntop(socket.AF_INET, self._read(client, 4)) - elif atyp == 4: - host = socket.inet_ntop(socket.AF_INET6, self._read(client, 16)) - elif atyp == 3: - length = self._read(client, 1)[0] - host = self._read(client, length).decode("idna") - else: - client.sendall(b"\x05\x08\x00\x01\x00\x00\x00\x00\x00\x00") - return - port = int.from_bytes(self._read(client, 2), "big") - if version != 5 or command != 1 or reserved != 0: - client.sendall(b"\x05\x07\x00\x01\x00\x00\x00\x00\x00\x00") - return - target = self.mapping.get((host.lower(), port), (host, port)) - try: - upstream = socket.create_connection(target, timeout=5) - except OSError: - client.sendall(b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00") - return - bound_host, bound_port = upstream.getsockname()[:2] - try: - bound_ip = socket.inet_pton(socket.AF_INET, bound_host) - atyp = 1 - except OSError: - bound_ip = socket.inet_pton(socket.AF_INET6, bound_host) - atyp = 4 - client.sendall(bytes((5, 0, 0, atyp)) + bound_ip + int(bound_port).to_bytes(2, "big")) - client.setblocking(False) - upstream.setblocking(False) - while True: - readable, _, _ = select.select([client, upstream], [], [], 10) - if not readable: - continue - for source in readable: - data = source.recv(65536) - if not data: - return - target_sock = upstream if source is client else client - target_sock.sendall(data) - except (OSError, EOFError): - pass - finally: - if upstream is not None: - upstream.close() - client.close() - - def _serve(self) -> None: - while not self._closed.is_set(): - try: - client, _ = self.listener.accept() - except socket.timeout: - continue - except OSError: - break - thread = threading.Thread(target=self._serve_client, args=(client,), daemon=True) - self._threads.append(thread) - thread.start() - - def close(self) -> None: - self._closed.set() - self.listener.close() - self._thread.join(timeout=2) - for thread in self._threads: - thread.join(timeout=2) diff --git a/test/test_socks5.py b/test/test_socks5.py index 52b7ec1a8b..f4989137e1 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -17,7 +17,7 @@ from __future__ import annotations import asyncio -import select +import os import socket import ssl import sys @@ -27,17 +27,26 @@ from types import SimpleNamespace from typing import Any, Optional, cast from unittest import mock +from urllib.parse import quote_plus import pytest sys.path[0:0] = [""] +from pymongo import monitoring from pymongo.encryption_options import _HAVE_PYMONGOCRYPT +from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.pool_shared import _configured_socket, _create_connection, _Socks5Error -from test import PyMongoTestCase, unittest -from test.helpers_shared import CERT_PATH -from test.socks5_helpers import Socks5Server, _Socks5Server +from test import ( + IntegrationTest, + PyMongoTestCase, + client_context, + unittest, +) +from test.helpers_shared import CERT_PATH, TLS_OPTIONS +from test.socks5_helpers import _Socks5Server +from test.utils import wait_until _IS_SYNC = True pytestmark = pytest.mark.default @@ -69,6 +78,11 @@ def sock_connect(sock: socket.socket, target: tuple[str, int]) -> None: class TestSocks5(PyMongoTestCase): + def _create_server(self, **kwargs: Any) -> _Socks5Server: + server = _Socks5Server(**kwargs) + self.addCleanup(server.close) + return server + def _connect( self, server: _Socks5Server, @@ -77,7 +91,7 @@ def _connect( password: Optional[str] = None, timeout: float = 2, ) -> socket.socket: - return _create_connection( + sock = _create_connection( address, PoolOptions( connect_timeout=timeout, @@ -86,9 +100,11 @@ def _connect( ), ), ) + self.addCleanup(sock.close) + return sock def test_greeting_and_authenticated_connect(self) -> None: - server = _Socks5Server(method=2) + server = self._create_server(method=2) sock = self._connect(server, ("mongodb.example", 27017), "üser", "påss") sock.close() server.close() @@ -101,7 +117,7 @@ def test_greeting_and_authenticated_connect(self) -> None: ) def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: - server = _Socks5Server(method=0) + server = self._create_server(method=0) sock = self._connect(server, ("127.0.0.1", 27017), "user", "password") sock.close() server.close() @@ -110,7 +126,7 @@ def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: self.assertEqual(server.request, b"\x05\x01\x00\x01\x7f\x00\x00\x01\x69\x89") def test_no_credentials_only_offer_no_authentication(self) -> None: - server = _Socks5Server(method=2) + server = self._create_server(method=2) with self.assertRaises(_Socks5Error): self._connect(server, ("localhost", 27017)) server.close() @@ -118,13 +134,13 @@ def test_no_credentials_only_offer_no_authentication(self) -> None: def test_rejects_all_unacceptable_methods(self) -> None: for method in (0x01, 0xFF): - server = _Socks5Server(method=method) + server = self._create_server(method=method) with self.subTest(method=method), self.assertRaises(_Socks5Error): self._connect(server, ("localhost", 27017)) server.close() def test_rejects_authentication_failure(self) -> None: - server = _Socks5Server(method=2, auth_status=1) + server = self._create_server(method=2, auth_status=1) with self.assertRaises(_Socks5Error): self._connect(server, ("localhost", 27017), "user", "password") server.close() @@ -142,7 +158,7 @@ def test_connect_request_address_types(self) -> None: ), ) for host, expected in cases: - server = _Socks5Server() + server = self._create_server() sock = self._connect(server, (host, 27017)) sock.close() server.close() @@ -150,7 +166,7 @@ def test_connect_request_address_types(self) -> None: def test_drains_bound_address_and_preserves_early_payload(self) -> None: for atyp in (1, 3, 4): - server = _Socks5Server(bound_atyp=atyp, payload=b"mongo") + server = self._create_server(bound_atyp=atyp, payload=b"mongo") sock = self._connect(server, ("localhost", 27017)) if _IS_SYNC: payload = sock.recv(5) @@ -163,7 +179,7 @@ def test_drains_bound_address_and_preserves_early_payload(self) -> None: def test_rejects_socks_reply_codes(self) -> None: for reply in range(1, 9): - server = _Socks5Server(reply=reply) + server = self._create_server(reply=reply) with self.subTest(reply=reply), self.assertRaises(_Socks5Error): self._connect(server, ("localhost", 27017)) server.close() @@ -217,15 +233,15 @@ def test_client_equality_includes_proxy_options(self) -> None: def test_bad_versions_and_unexpected_eof_fail(self) -> None: cases = ( - (_Socks5Server(method_version=4), ("localhost", 27017), None), - (_Socks5Server(reply_version=4), ("localhost", 27017), None), - (_Socks5Server(eof_stage="greeting"), ("localhost", 27017), None), + (self._create_server(method_version=4), ("localhost", 27017), None), + (self._create_server(reply_version=4), ("localhost", 27017), None), + (self._create_server(eof_stage="greeting"), ("localhost", 27017), None), ( - _Socks5Server(method=2, eof_stage="auth"), + self._create_server(method=2, eof_stage="auth"), ("localhost", 27017), ("user", "password"), ), - (_Socks5Server(eof_stage="connect"), ("localhost", 27017), None), + (self._create_server(eof_stage="connect"), ("localhost", 27017), None), ) for server, address, credentials in cases: with self.subTest(stage=server.eof_stage, version=server.method_version): @@ -234,25 +250,23 @@ def test_bad_versions_and_unexpected_eof_fail(self) -> None: server.close() def test_timeout_covers_the_entire_handshake(self) -> None: - server = _Socks5Server(stall=True) + server = self._create_server(stall=True) with self.assertRaises(socket.timeout): self._connect(server, ("localhost", 27017), timeout=0.1) server.close() @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") def test_cancellation_closes_proxy_socket(self) -> None: - server = _Socks5Server(stall=True) - self.addCleanup(server.close) + server = self._create_server(stall=True) task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) - self.assertTrue(asyncio.to_thread(server.started.wait, 2)) + wait_until(server.started.is_set, "receive the SOCKS5 greeting", timeout=2) task.cancel() with self.assertRaises(asyncio.CancelledError): task @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") def test_cancellation_when_proxy_send_completes_closes_socket(self) -> None: - server = _Socks5Server(stall=True) - self.addCleanup(server.close) + server = self._create_server(stall=True) loop = asyncio.get_running_loop() original_sendall = loop.sock_sendall sockets: list[socket.socket] = [] @@ -269,16 +283,57 @@ def cancel_after_send(sock: socket.socket, data: bytes) -> None: self.assertEqual(len(sockets), 1) self.assertEqual(sockets[0].fileno(), -1) + @client_context.require_async # type: ignore[untyped-decorator] def test_tls_sni_targets_destination_not_proxy(self) -> None: - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - target_address = listener.getsockname() tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") sni_names: list[Optional[str]] = [] tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + def serve_tls(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + reader.read(1) + finally: + writer.close() + writer.wait_closed() + + tls_server: Any = asyncio.start_server(serve_tls, "127.0.0.1", 0, ssl=tls_context) + + def close_tls_server() -> None: + tls_server.close() + tls_server.wait_closed() + + self.addCleanup(close_tls_server) + tls_sockets = tls_server.sockets + assert tls_sockets is not None + target_address = tls_sockets[0].getsockname() + proxy = self._create_server(target=target_address) + client_context = ssl.create_default_context() + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + sock = _configured_socket( + ("mongodb.example", target_address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, client_context), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), + ), + ) + self.addCleanup(sock.close) + sock.close() + self.assertEqual(sni_names, ["mongodb.example"]) + + @client_context.require_sync # type: ignore[untyped-decorator] + def test_tls_sni_targets_destination_not_proxy_sync(self) -> None: + tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") + sni_names: list[Optional[str]] = [] + tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + target_address = listener.getsockname() + def serve_tls() -> None: try: raw, _ = listener.accept() @@ -291,7 +346,13 @@ def serve_tls() -> None: tls_thread = threading.Thread(target=serve_tls, daemon=True) tls_thread.start() - proxy = _Socks5Server(target=target_address) + + def close_tls_server() -> None: + listener.close() + tls_thread.join(timeout=3) + + self.addCleanup(close_tls_server) + proxy = self._create_server(target=target_address) client_context = ssl.create_default_context() client_context.check_hostname = False client_context.verify_mode = ssl.CERT_NONE @@ -303,9 +364,8 @@ def serve_tls() -> None: proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), ), ) + self.addCleanup(sock.close) sock.close() - proxy.close() - tls_thread.join(timeout=3) self.assertEqual(sni_names, ["mongodb.example"]) @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") @@ -323,7 +383,13 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: client = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) expected = _ProxyOptions("127.0.0.1") - io = _EncryptionIO(client, client.keyvault.datakeys, None, FakeOpts()) # type: ignore[arg-type] + io = _EncryptionIO( + client, + client.keyvault.datakeys, + None, + FakeOpts(), # type: ignore[arg-type] + proxy_client=client, + ) connection = mock.MagicMock() def connect(_address, options, _callback, _timeout): @@ -444,8 +510,7 @@ def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: self.assertFalse(wrapped.has_error_label("RetryableError")) def test_multi_address_proxy_fallback_on_connect_error(self) -> None: - server = _Socks5Server() - self.addCleanup(server.close) + server = self._create_server() dead_addr = ("127.0.0.1", 1) fake_results = [ @@ -502,15 +567,8 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: parent_client.close() kv_client.close() - def test_socks5_helper_edge_cases(self) -> None: - from pymongo.common import validate_proxy_host, validate_proxy_port - from pymongo.pool_shared import ( - _proxy_deadline, - _proxy_remaining, - _socks5_check_connect_reply, - _socks5_request, - _Socks5Timeout, - ) + def test_proxy_host_validation(self) -> None: + from pymongo.common import validate_proxy_host self.assertIsNone(validate_proxy_host("proxyHost", None)) self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") @@ -518,10 +576,17 @@ def test_socks5_helper_edge_cases(self) -> None: validate_proxy_host("proxyHost", "[not-ipv6]") with self.assertRaises(ValueError): validate_proxy_host("proxyHost", "bad/host") + + def test_proxy_port_validation(self) -> None: + from pymongo.common import validate_proxy_port + self.assertIsNone(validate_proxy_port("proxyPort", None)) with self.assertRaises(TypeError): validate_proxy_port("proxyPort", True) + def test_proxy_deadline(self) -> None: + from pymongo.pool_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout + self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) self.assertIsNone(_proxy_remaining(None)) with self.assertRaises(_Socks5Timeout): @@ -532,14 +597,43 @@ def test_socks5_helper_edge_cases(self) -> None: ): _proxy_deadline(PoolOptions(connect_timeout=1)) + def test_proxy_socket_timeout_rejects_expired_deadline(self) -> None: + from pymongo.pool_shared import _proxy_socket_timeout, _Socks5Timeout + + with ( + mock.patch("pymongo.pool_shared._csot.get_timeout", return_value=True), + mock.patch("pymongo.pool_shared._csot.remaining", return_value=-0.1), + self.assertRaises(_Socks5Timeout), + ): + _proxy_socket_timeout(PoolOptions(connect_timeout=1)) + + def test_socks5_request_rejects_long_domain(self) -> None: + from pymongo.pool_shared import _socks5_request + with self.assertRaisesRegex(_Socks5Error, "invalid domain name length"): _socks5_request(("a" * 256, 27017)) + + def test_socks5_request_rejects_invalid_domain(self) -> None: + from pymongo.pool_shared import _socks5_request + with self.assertRaisesRegex(_Socks5Error, "invalid domain name"): _socks5_request((".münich.example", 27017)) + + def test_socks5_request_checks_port_before_encoding_domain(self) -> None: + from pymongo.pool_shared import _socks5_request + with self.assertRaisesRegex(_Socks5Error, "destination port is required"): - _socks5_request(("localhost", None)) # type: ignore[arg-type] + _socks5_request(("a" * 256, None)) # type: ignore[arg-type] + + def test_socks5_connect_reply_rejects_invalid_reserved_byte(self) -> None: + from pymongo.pool_shared import _socks5_check_connect_reply + with self.assertRaisesRegex(_Socks5Error, "invalid reserved byte"): _socks5_check_connect_reply(b"\x05\x00\x01\x01") + + def test_socks5_connect_reply_rejects_invalid_address_type(self) -> None: + from pymongo.pool_shared import _socks5_check_connect_reply + with self.assertRaisesRegex(_Socks5Error, "invalid address type"): _socks5_check_connect_reply(b"\x05\x00\x00\x09") @@ -586,149 +680,136 @@ def dead_async(*_a: Any, **_kw: Any) -> Any: ): _create_connection(("localhost", 27017), proxy_opts) - def test_prose_client_options(self) -> None: - from pymongo.errors import OperationFailure - from pymongo.synchronous.mongo_client import MongoClient - from test import test_socks5_prose - def command(name: str) -> None: - if name == "nonexistent_socks5_test_command": - raise OperationFailure("unsupported command") +class _CommandListener(monitoring.CommandListener): + def __init__(self) -> None: + self.connection_ids: list[tuple[str, Optional[int]]] = [] - def close() -> None: - pass + def started(self, event: monitoring.CommandStartedEvent) -> None: + self.connection_ids.append(event.connection_id) - fake_client = SimpleNamespace(admin=SimpleNamespace(command=command), close=close) - case = test_socks5_prose.TestSocks5Prose() - case.credentials = {"username": "dbuser", "password": "dbpass", "authSource": "admin"} - listener = test_socks5_prose._CommandListener() - for tls in (False, True): - for use_uri in (False, True): - for event_listener in (None, listener): - with self.subTest(tls=tls, use_uri=use_uri, listener=event_listener): - with ( - mock.patch.object(test_socks5_prose.client_context, "tls", tls), - mock.patch.object( - test_socks5_prose, "MongoClient", return_value=fake_client - ) as factory, - ): - case._run_case( - "localhost:12345", - 1080, - ("proxyuser", "proxypass"), - True, - use_uri, - event_listener, - ) - factory.assert_called_once() - args, kwargs = factory.call_args - client = MongoClient(*args, connect=False, **kwargs) - try: - self.assertEqual( - client.options.pool_options._proxy_options, - _ProxyOptions("localhost", 1080, "proxyuser", "proxypass"), - ) - self.assertEqual(client.options.direct_connection, True) - self.assertEqual(client.options.server_selection_timeout, 2.5) - self.assertEqual( - client.options.pool_options._ssl_context is not None, tls - ) - self.assertEqual(kwargs["authSource"], "admin") - if event_listener is None: - self.assertNotIn("event_listeners", kwargs) - else: - self.assertEqual(kwargs["event_listeners"], [event_listener]) - finally: - client.close() - - def test_in_process_socks5_server_forwarding(self) -> None: - echo_listener = socket.socket() - echo_listener.bind(("127.0.0.1", 0)) - echo_listener.listen(2) - echo_addr = echo_listener.getsockname() - - def serve_echo() -> None: - for _ in range(2): - try: - conn, _ = echo_listener.accept() - except OSError: - return - with conn: - data = conn.recv(64) - if data: - conn.sendall(b"ack:" + data) - - echo_thread = threading.Thread(target=serve_echo, daemon=True) - echo_thread.start() - self.addCleanup(echo_thread.join, 2) - self.addCleanup(echo_listener.close) - - mapping = {("mapped.example", 12345): echo_addr} - auth_proxy = Socks5Server(0, mapping, ("user", "secret")) - noauth_proxy = Socks5Server(0, mapping) - self.addCleanup(auth_proxy.close) - self.addCleanup(noauth_proxy.close) - - sock1 = _create_connection( - ("mapped.example", 12345), - PoolOptions( - connect_timeout=2, - proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "secret"), - ), - ) - try: - if _IS_SYNC: - sock1.settimeout(2) - sock1.sendall(b"hello") - response = sock1.recv(64) - else: - sock1.setblocking(False) - loop = asyncio.get_running_loop() - loop.sock_sendall(sock1, b"hello") - response = asyncio.wait_for(loop.sock_recv(sock1, 64), timeout=2) - self.assertEqual(response, b"ack:hello") - finally: - sock1.close() + def succeeded(self, event: monitoring.CommandSucceededEvent) -> None: + self.connection_ids.append(event.connection_id) - sock2 = _create_connection( - echo_addr, - PoolOptions( - connect_timeout=2, - proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), - ), + def failed(self, event: monitoring.CommandFailedEvent) -> None: + self.connection_ids.append(event.connection_id) + + +class TestSocks5Prose(IntegrationTest): + @client_context.require_replica_set + def setUp(self) -> None: + if not os.environ.get("SOCKS5_PROXIES_STARTED"): + self.skipTest("SOCKS5 proxies must be started by the Evergreen SOCKS5 tasks.") + super().setUp() + self.nodes = sorted(client_context.nodes) + self.credentials = getattr(self, "credentials", {}) + self.auth_port = 1080 + self.noauth_port = 1081 + + def _hosts(self, mapped: bool) -> str: + if mapped: + return "localhost:12345" + return ",".join( + f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" + for host, port in self.nodes ) + + def _run_case( + self, + hosts: str, + port: int, + credentials: Optional[tuple[str, str]], + succeeds: bool, + use_uri: bool, + listener: Optional[_CommandListener] = None, + ) -> None: + from pymongo.synchronous.mongo_client import MongoClient + + proxy_options = {"proxyHost": "localhost", "proxyPort": port} + if credentials: + proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials + tls_options = dict(TLS_OPTIONS) if client_context.tls else {} + options = dict(tls_options) + options.update(self.credentials) + options["serverSelectionTimeoutMS"] = 2500 + if hosts == "localhost:12345": + options["directConnection"] = True + if listener is not None: + options["event_listeners"] = [listener] + if use_uri: + uri_options = "&".join( + f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() + ) + hosts = f"mongodb://{hosts}/?{uri_options}" + else: + options.update(proxy_options) + client = MongoClient(hosts, **options) try: - if _IS_SYNC: - sock2.settimeout(2) - sock2.sendall(b"world") - response = sock2.recv(64) + if succeeds: + client.admin.command(HelloCompat.CMD) + if listener is not None: + from pymongo.errors import OperationFailure + + with self.assertRaises(OperationFailure): + client.admin.command("nonexistent_socks5_test_command") else: - sock2.setblocking(False) - loop = asyncio.get_running_loop() - loop.sock_sendall(sock2, b"world") - response = asyncio.wait_for(loop.sock_recv(sock2, 64), timeout=2) - self.assertEqual(response, b"ack:world") + from pymongo.errors import ConnectionFailure + + with self.assertRaises(ConnectionFailure): + client.admin.command(HelloCompat.CMD) finally: - sock2.close() + client.close() - with self.assertRaises(_Socks5Error): - _create_connection( - ("mapped.example", 12345), - PoolOptions( - connect_timeout=2, - proxy_options=_ProxyOptions("127.0.0.1", auth_proxy.port, "user", "wrong"), - ), - ) - with ( - mock.patch( - "socket.create_connection", side_effect=ConnectionRefusedError("connection refused") - ), - self.assertRaisesRegex(_Socks5Error, "connection refused"), - ): - _create_connection( - ("127.0.0.1", 1), - PoolOptions( - connect_timeout=2, - proxy_options=_ProxyOptions("127.0.0.1", noauth_proxy.port), - ), - ) + def test_connection_string_prose_table(self) -> None: + cases = ( + (True, self.auth_port, None, False), + (True, self.noauth_port, None, True), + (False, self.auth_port, None, False), + (False, self.noauth_port, None, True), + (True, self.auth_port, ("nonexistentuser", "badauth"), False), + (True, self.noauth_port, ("nonexistentuser", "badauth"), True), + (False, self.noauth_port, ("nonexistentuser", "badauth"), True), + (True, self.auth_port, ("username", "p4ssw0rd"), True), + (True, self.noauth_port, None, True), + (False, self.auth_port, ("username", "p4ssw0rd"), True), + (False, self.noauth_port, None, True), + ) + mapped_listener = _CommandListener() + rs_listener = _CommandListener() + for index, (mapped, proxy_port, credentials, succeeds) in enumerate(cases): + for use_uri in (True, False): + listener = None + if not use_uri: + if index == 1: + listener = mapped_listener + elif index == 3: + listener = rs_listener + with self.subTest( + case=index, + mapped=mapped, + proxy_port=proxy_port, + credentials=credentials, + use_uri=use_uri, + ): + self._run_case( + self._hosts(mapped), + proxy_port, + credentials, + succeeds, + use_uri, + listener, + ) + proxy_addresses = { + ("localhost", self.auth_port), + ("localhost", self.noauth_port), + ("127.0.0.1", self.auth_port), + ("127.0.0.1", self.noauth_port), + } + self.assertGreaterEqual(len(mapped_listener.connection_ids), 4) + for connection_id in mapped_listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + self.assertEqual(connection_id, ("localhost", 12345)) + self.assertGreaterEqual(len(rs_listener.connection_ids), 4) + for connection_id in rs_listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + self.assertIn(connection_id, self.nodes) diff --git a/test/test_socks5_prose.py b/test/test_socks5_prose.py deleted file mode 100644 index da88f7f11f..0000000000 --- a/test/test_socks5_prose.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2026-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""SOCKS5 connection-string prose tests.""" - -from __future__ import annotations - -import os -import select -import socket -import threading -from typing import Optional -from urllib.parse import quote_plus - -import pytest - -from pymongo import monitoring -from pymongo.hello import HelloCompat -from pymongo.synchronous.mongo_client import MongoClient -from test import IntegrationTest, client_context -from test.helpers_shared import TLS_OPTIONS -from test.socks5_helpers import Socks5Server - -_IS_SYNC = True -pytestmark = pytest.mark.default - - -class _CommandListener(monitoring.CommandListener): - def __init__(self) -> None: - self.connection_ids: list[tuple[str, Optional[int]]] = [] - - def started(self, event: monitoring.CommandStartedEvent) -> None: - self.connection_ids.append(event.connection_id) - - def succeeded(self, event: monitoring.CommandSucceededEvent) -> None: - self.connection_ids.append(event.connection_id) - - def failed(self, event: monitoring.CommandFailedEvent) -> None: - self.connection_ids.append(event.connection_id) - - -class TestSocks5Prose(IntegrationTest): - @client_context.require_replica_set - def setUp(self) -> None: - super().setUp() - self.nodes = sorted(client_context.nodes) - self.mapping = {("localhost", 12345): self.nodes[0]} - self.credentials = getattr(self, "credentials", {}) - if os.environ.get("SOCKS5_PROXIES_STARTED"): - self.auth_port = 1080 - self.noauth_port = 1081 - else: - auth_proxy = Socks5Server(1080, self.mapping, ("username", "p4ssw0rd")) - noauth_proxy = Socks5Server(1081, self.mapping) - self.addCleanup(auth_proxy.close) - self.addCleanup(noauth_proxy.close) - self.auth_port = auth_proxy.port - self.noauth_port = noauth_proxy.port - - def _hosts(self, mapped: bool) -> str: - if mapped: - return "localhost:12345" - return ",".join( - f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" - for host, port in self.nodes - ) - - def _run_case( - self, - hosts: str, - port: int, - credentials: Optional[tuple[str, str]], - succeeds: bool, - use_uri: bool, - listener: Optional[_CommandListener] = None, - ) -> None: - proxy_options = {"proxyHost": "localhost", "proxyPort": port} - if credentials: - proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials - tls_options = dict(TLS_OPTIONS) if client_context.tls else {} - options = dict(tls_options) - options.update(self.credentials) - options["serverSelectionTimeoutMS"] = 2500 - if hosts == "localhost:12345": - options["directConnection"] = True - if listener is not None: - options["event_listeners"] = [listener] - if use_uri: - uri_options = "&".join( - f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() - ) - hosts = f"mongodb://{hosts}/?{uri_options}" - else: - options.update(proxy_options) - client = MongoClient(hosts, **options) - try: - if succeeds: - client.admin.command(HelloCompat.CMD) - if listener is not None: - from pymongo.errors import OperationFailure - - with self.assertRaises(OperationFailure): - client.admin.command("nonexistent_socks5_test_command") - else: - from pymongo.errors import ConnectionFailure - - with self.assertRaises(ConnectionFailure): - client.admin.command(HelloCompat.CMD) - finally: - client.close() - - def test_connection_string_prose_table(self) -> None: - cases = ( - (True, self.auth_port, None, False), - (True, self.noauth_port, None, True), - (False, self.auth_port, None, False), - (False, self.noauth_port, None, True), - (True, self.auth_port, ("nonexistentuser", "badauth"), False), - (True, self.noauth_port, ("nonexistentuser", "badauth"), True), - (False, self.noauth_port, ("nonexistentuser", "badauth"), True), - (True, self.auth_port, ("username", "p4ssw0rd"), True), - (True, self.noauth_port, None, True), - (False, self.auth_port, ("username", "p4ssw0rd"), True), - (False, self.noauth_port, None, True), - ) - mapped_listener = _CommandListener() - rs_listener = _CommandListener() - for index, (mapped, proxy_port, credentials, succeeds) in enumerate(cases): - for use_uri in (True, False): - listener = None - if not use_uri: - if index == 1: - listener = mapped_listener - elif index == 3: - listener = rs_listener - self._run_case( - self._hosts(mapped), - proxy_port, - credentials, - succeeds, - use_uri, - listener, - ) - proxy_addresses = { - ("localhost", self.auth_port), - ("localhost", self.noauth_port), - ("127.0.0.1", self.auth_port), - ("127.0.0.1", self.noauth_port), - } - self.assertGreaterEqual(len(mapped_listener.connection_ids), 4) - for connection_id in mapped_listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - self.assertEqual(connection_id, ("localhost", 12345)) - self.assertGreaterEqual(len(rs_listener.connection_ids), 4) - for connection_id in rs_listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - self.assertIn(connection_id, self.nodes) From ca0a4cab8fe0ec95c23d37e04d6f5d03e42616f1 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 19:48:32 +0000 Subject: [PATCH 09/16] PYTHON-2943 Run SOCKS5 prose tests in PR builds Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .evergreen/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c7dcfbb49f..f12e5b43e0 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -46,6 +46,7 @@ post: tasks: - name: test-socks5-prose-noauth-nossl + tags: [pr] commands: - func: run server vars: @@ -113,6 +114,7 @@ tasks: - MONGODB_URI - name: test-socks5-prose-noauth-ssl + tags: [pr] commands: - func: run server vars: @@ -221,6 +223,7 @@ buildvariants: display_name: "SOCKS5 Prose Tests" run_on: rhel8.10-small patchable: true + tags: [pr] expansions: SOCKS5_PROXIES_STARTED: "1" UV_PYTHON: "3.11" From 7a3ec6f21dbbbbdaa06b62996a907061ce533be3 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 19:59:51 +0000 Subject: [PATCH 10/16] PYTHON-2943 Use supported Python for SOCKS5 tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .evergreen/config.yml | 16 ++++++++-------- .evergreen/scripts/run_socks5_proxy.sh | 6 ++++++ 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 .evergreen/scripts/run_socks5_proxy.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f12e5b43e0..9bb5b9e4cd 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -58,9 +58,9 @@ tasks: - command: subprocess.exec type: test params: - binary: python3 + binary: bash args: - - .evergreen/scripts/run_socks5_proxy.py + - .evergreen/scripts/run_socks5_proxy.sh - --port - "1080" - --auth @@ -73,9 +73,9 @@ tasks: - command: subprocess.exec type: test params: - binary: python3 + binary: bash args: - - .evergreen/scripts/run_socks5_proxy.py + - .evergreen/scripts/run_socks5_proxy.sh - --port - "1081" background: true @@ -126,9 +126,9 @@ tasks: - command: subprocess.exec type: test params: - binary: python3 + binary: bash args: - - .evergreen/scripts/run_socks5_proxy.py + - .evergreen/scripts/run_socks5_proxy.sh - --port - "1080" - --auth @@ -141,9 +141,9 @@ tasks: - command: subprocess.exec type: test params: - binary: python3 + binary: bash args: - - .evergreen/scripts/run_socks5_proxy.py + - .evergreen/scripts/run_socks5_proxy.sh - --port - "1081" background: true diff --git a/.evergreen/scripts/run_socks5_proxy.sh b/.evergreen/scripts/run_socks5_proxy.sh new file mode 100644 index 0000000000..68e7d507b0 --- /dev/null +++ b/.evergreen/scripts/run_socks5_proxy.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -eu + +source "${DRIVERS_TOOLS}/.evergreen/find-python3.sh" +PYTHON_BINARY="$(ensure_python3 2>/dev/null)" +exec "$PYTHON_BINARY" "$(dirname "${BASH_SOURCE[0]}")/run_socks5_proxy.py" "$@" From a150960b435766c191b5d93b3c933d1b88182592 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Wed, 4 Nov 2026 20:33:33 +0000 Subject: [PATCH 11/16] PYTHON-2943 Fix Evergreen KMS test failures Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .evergreen/scripts/run_socks5_proxy.sh | 0 test/test_kms_connect.py | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) mode change 100644 => 100755 .evergreen/scripts/run_socks5_proxy.sh diff --git a/.evergreen/scripts/run_socks5_proxy.sh b/.evergreen/scripts/run_socks5_proxy.sh old mode 100644 new mode 100755 diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index f5e79b5048..c9b5d84772 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -1068,7 +1068,9 @@ def action(context): return "not-a-socket" opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=api.callback(action)) - io = api.encryption()._EncryptionIO(None, mock.MagicMock(), None, opts) + proxy_client = mock.MagicMock() + proxy_client.options.pool_options._proxy_options = None + io = api.encryption()._EncryptionIO(None, mock.MagicMock(), None, opts, proxy_client) class StubKmsContext: endpoint = "kms.example.com:443" From fa5a3fe6276b467612b0218015cfb2556260e6e1 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Thu, 5 Nov 2026 16:06:47 +0000 Subject: [PATCH 12/16] PYTHON-2943 Refactor SOCKS5 support and Evergreen integration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .evergreen/config.yml | 148 -------- .evergreen/generated_configs/tasks.yml | 30 ++ .evergreen/generated_configs/variants.yml | 11 + .evergreen/scripts/generate_config.py | 35 ++ .evergreen/scripts/run_socks5_proxy.py | 38 +++ .evergreen/scripts/run_socks5_proxy.sh | 6 - .evergreen/scripts/run_tests.py | 5 + .evergreen/scripts/setup_tests.py | 71 ++++ .evergreen/scripts/teardown_tests.py | 5 + .evergreen/scripts/test_generate_config.py | 30 ++ .evergreen/scripts/test_socks5_proxy.py | 169 ++++++++++ .evergreen/scripts/utils.py | 1 + CONTRIBUTING.md | 10 + pymongo/asynchronous/encryption.py | 14 +- pymongo/asynchronous/socks5_support.py | 141 ++++++++ pymongo/pool_shared.py | 371 +-------------------- pymongo/socks5_shared.py | 258 ++++++++++++++ pymongo/synchronous/encryption.py | 14 +- pymongo/synchronous/socks5_support.py | 137 ++++++++ test/asynchronous/test_socks5.py | 103 +++--- test/test_socks5.py | 103 +++--- tools/synchro.py | 7 + 22 files changed, 1080 insertions(+), 627 deletions(-) delete mode 100755 .evergreen/scripts/run_socks5_proxy.sh create mode 100644 .evergreen/scripts/test_socks5_proxy.py create mode 100644 pymongo/asynchronous/socks5_support.py create mode 100644 pymongo/socks5_shared.py create mode 100644 pymongo/synchronous/socks5_support.py diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9bb5b9e4cd..791dd332f0 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -45,142 +45,6 @@ post: - func: "cleanup" tasks: - - name: test-socks5-prose-noauth-nossl - tags: [pr] - commands: - - func: run server - vars: - TEST_NAME: default - VERSION: "8.0" - TOPOLOGY: replica_set - AUTH: noauth - SSL: nossl - - command: subprocess.exec - type: test - params: - binary: bash - args: - - .evergreen/scripts/run_socks5_proxy.sh - - --port - - "1080" - - --auth - - username:p4ssw0rd - background: true - working_dir: src - include_expansions_in_env: - - MONGODB_URI - - DRIVERS_TOOLS - - command: subprocess.exec - type: test - params: - binary: bash - args: - - .evergreen/scripts/run_socks5_proxy.sh - - --port - - "1081" - background: true - working_dir: src - include_expansions_in_env: - - MONGODB_URI - - DRIVERS_TOOLS - - command: subprocess.exec - type: test - params: - binary: bash - args: - - .evergreen/just.sh - - setup-tests - - default - working_dir: src - include_expansions_in_env: - - AUTH - - SSL - - MONGODB_URI - - UV_PYTHON - - command: subprocess.exec - type: test - params: - binary: bash - args: - - .evergreen/just.sh - - run-tests - - test/test_socks5.py - - test/asynchronous/test_socks5.py - - -k - - TestSocks5Prose - working_dir: src - include_expansions_in_env: - - SOCKS5_PROXIES_STARTED - - MONGODB_URI - - - name: test-socks5-prose-noauth-ssl - tags: [pr] - commands: - - func: run server - vars: - TEST_NAME: default - VERSION: "8.0" - TOPOLOGY: replica_set - AUTH: noauth - SSL: ssl - - command: subprocess.exec - type: test - params: - binary: bash - args: - - .evergreen/scripts/run_socks5_proxy.sh - - --port - - "1080" - - --auth - - username:p4ssw0rd - background: true - working_dir: src - include_expansions_in_env: - - MONGODB_URI - - DRIVERS_TOOLS - - command: subprocess.exec - type: test - params: - binary: bash - args: - - .evergreen/scripts/run_socks5_proxy.sh - - --port - - "1081" - background: true - working_dir: src - include_expansions_in_env: - - MONGODB_URI - - DRIVERS_TOOLS - - command: subprocess.exec - type: test - params: - binary: bash - args: - - .evergreen/just.sh - - setup-tests - - default - working_dir: src - include_expansions_in_env: - - AUTH - - SSL - - MONGODB_URI - - UV_PYTHON - - command: subprocess.exec - type: test - params: - binary: bash - args: - - .evergreen/just.sh - - run-tests - - test/test_socks5.py - - test/asynchronous/test_socks5.py - - -k - - TestSocks5Prose - working_dir: src - include_expansions_in_env: - - SOCKS5_PROXIES_STARTED - - MONGODB_URI - - name: linkcheck commands: - command: subprocess.exec @@ -218,15 +82,3 @@ buildvariants: patchable: true tasks: - name: resync_specs - - - name: socks5-prose - display_name: "SOCKS5 Prose Tests" - run_on: rhel8.10-small - patchable: true - tags: [pr] - expansions: - SOCKS5_PROXIES_STARTED: "1" - UV_PYTHON: "3.11" - tasks: - - name: test-socks5-prose-noauth-nossl - - name: test-socks5-prose-noauth-ssl diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 8e2214c680..ab3818ffdd 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -3778,6 +3778,36 @@ tasks: - pr - "9.0" + # Socks5 tests + - name: test-socks5-prose-noauth-nossl + commands: + - func: run server + vars: + TEST_NAME: socks5 + VERSION: "9.0" + TOPOLOGY: replica_set + AUTH: noauth + SSL: nossl + - func: run tests + vars: + TEST_NAME: socks5 + COVERAGE: "1" + tags: [socks5, pr] + - name: test-socks5-prose-noauth-ssl + commands: + - func: run server + vars: + TEST_NAME: socks5 + VERSION: "9.0" + TOPOLOGY: replica_set + AUTH: noauth + SSL: ssl + - func: run tests + vars: + TEST_NAME: socks5 + COVERAGE: "1" + tags: [socks5, pr] + # Standard tests - name: test-standard-v4.4-python3.12-sync-noauth-ssl-replica-set commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 67d069f438..5f53ae288a 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -631,6 +631,17 @@ buildvariants: TEST_NAME: sfp tags: [pr] + # Socks5 tests + - name: socks5-prose + tasks: + - name: .socks5 + display_name: SOCKS5 Prose Tests + run_on: + - rhel8.10-small + expansions: + UV_PYTHON: "3.11" + tags: [pr] + # Stable api tests - name: stable-api-require-v1-rhel8-auth tasks: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 3589a81c86..99fcfe0a37 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -558,6 +558,19 @@ def create_coverage_report_variants(): return [create_variant(["coverage-report"], "Coverage Report", host=DEFAULT_HOST)] +def create_socks5_variants(): + variant = create_variant( + [".socks5"], + "SOCKS5 Prose Tests", + host=DEFAULT_HOST, + tags=["pr"], + expansions=dict(UV_PYTHON=MIN_PYTHON), + patchable=True, + ) + variant.name = "socks5-prose" + return [variant] + + def create_kms_variants(): tasks = [] tasks.append(EvgTaskRef(name="test-gcpkms", batchtime=BATCHTIME_DAY)) @@ -972,6 +985,28 @@ def create_fuzz_tasks(): return tasks +def create_socks5_tasks(): + tasks = [] + for ssl in ["nossl", "ssl"]: + server_vars = dict( + TEST_NAME="socks5", + VERSION=MAX_VERSION, + TOPOLOGY="replica_set", + AUTH="noauth", + SSL=ssl, + ) + commands = [ + FunctionCall(func="run server", vars=server_vars), + FunctionCall(func="run tests", vars=dict(TEST_NAME="socks5", COVERAGE="1")), + ] + tasks.append( + EvgTask( + name=f"test-socks5-prose-noauth-{ssl}", tags=["socks5", "pr"], commands=commands + ) + ) + return tasks + + def create_kms_tasks(): tasks = [] for kms_type in ["gcp", "azure"]: diff --git a/.evergreen/scripts/run_socks5_proxy.py b/.evergreen/scripts/run_socks5_proxy.py index 98cf45a982..ca8e335c76 100644 --- a/.evergreen/scripts/run_socks5_proxy.py +++ b/.evergreen/scripts/run_socks5_proxy.py @@ -3,10 +3,16 @@ import argparse import os import runpy +import shlex +import signal +import subprocess import sys from pathlib import Path +from shutil import which from urllib.parse import urlsplit +from utils import LOGGER + def _first_seed(uri: str) -> str: authority = urlsplit(uri).netloc.rsplit("@", 1)[-1] @@ -29,6 +35,38 @@ def _first_seed(uri: str) -> str: return f"[{host}]:{port}" if ":" in host else f"{host}:{port}" +def stop_socks5_proxies(pids: list[int]) -> None: + """Stop the proxy processes started for this checkout. + + :param pids: Process IDs recorded by the test setup. + :raises ValueError: If a process ID is not positive. + :raises RuntimeError: If ``ps`` is missing or a process is unrelated. + """ + ps = which("ps") + if ps is None: + raise RuntimeError("Cannot locate ps to verify SOCKS5 proxy process ownership") + script = str(Path(__file__).absolute()) + for pid in pids: + if pid <= 0: + raise ValueError(f"Invalid SOCKS5 proxy PID: {pid}") + result = subprocess.run( # noqa: S603 + [ps, "-ww", "-p", str(pid), "-o", "args="], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 1: + LOGGER.info("SOCKS5 proxy PID %s already exited", pid) + continue + result.check_returncode() + if script not in shlex.split(result.stdout): + raise RuntimeError(f"Refusing to stop unrelated process with PID {pid}") + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + LOGGER.info("SOCKS5 proxy PID %s already exited", pid) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--port", type=int, required=True) diff --git a/.evergreen/scripts/run_socks5_proxy.sh b/.evergreen/scripts/run_socks5_proxy.sh deleted file mode 100755 index 68e7d507b0..0000000000 --- a/.evergreen/scripts/run_socks5_proxy.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -eu - -source "${DRIVERS_TOOLS}/.evergreen/find-python3.sh" -PYTHON_BINARY="$(ensure_python3 2>/dev/null)" -exec "$PYTHON_BINARY" "$(dirname "${BASH_SOURCE[0]}")/run_socks5_proxy.py" "$@" diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 9c3cdf13f5..632faaf3f8 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -185,6 +185,11 @@ def handle_fuzz() -> None: def run() -> None: + if TEST_NAME == "socks5": + TEST_ARGS.extend( + ["test/test_socks5.py", "test/asynchronous/test_socks5.py", "-k", "TestSocks5Prose"] + ) + # Add diagnostic for python version. print("Running with python", sys.version) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index bfa767e342..25c8e0b2b9 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -4,8 +4,11 @@ import os import platform import shutil +import socket import stat +import subprocess import sys +import time import zipfile from pathlib import Path from urllib import request @@ -177,6 +180,71 @@ def get_secrets(name: str) -> dict[str, str]: return load_config_from_file(secrets_dir / "secrets-export.sh") +def setup_socks5() -> None: + """Start and verify the authenticated and unauthenticated test proxies. + + :raises ValueError: If ``MONGODB_URI`` is unset. + :raises OSError: If a proxy port is already in use. + :raises RuntimeError: If the tools checkout is missing or a proxy fails to start. + """ + check_drivers_tools() + if not os.environ.get("MONGODB_URI"): + raise ValueError("MONGODB_URI must point to the replica set for SOCKS5 tests") + + ports = (1080, 1081) + for port in ports: + with socket.socket() as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + + env = os.environ.copy() + env["DRIVERS_TOOLS"] = DRIVERS_TOOLS + processes: list[subprocess.Popen[bytes]] = [] + started = False + try: + for port in ports: + args = [sys.executable, str(HERE / "run_socks5_proxy.py"), "--port", str(port)] + if port == 1080: + args.extend(("--auth", "username:p4ssw0rd")) + log_path = HERE / f"socks5-proxy-{port}.log" + LOGGER.info("Starting SOCKS5 proxy on port %s; log: %s", port, log_path) + with log_path.open("wb") as log: + process = subprocess.Popen( # noqa: S603 + args, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + processes.append(process) + deadline = time.monotonic() + 10 + while True: + if process.poll() is not None: + raise RuntimeError(f"SOCKS5 proxy on port {port} exited; see {log_path}") + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.1): + break + except OSError as exc: + if time.monotonic() >= deadline: + raise RuntimeError( + f"SOCKS5 proxy on port {port} did not start; see {log_path}" + ) from exc + time.sleep(0.1) + write_env("SOCKS5_PROXY_PIDS", " ".join(str(process.pid) for process in processes)) + write_env("SOCKS5_PROXIES_STARTED") + started = True + finally: + if not started: + for process in processes: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + def handle_test_env() -> None: opts, _ = get_test_options("Set up the test environment and services.") test_name = opts.test_name @@ -335,6 +403,9 @@ def handle_test_env() -> None: cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start' run_command(cmd) + if test_name == "socks5": + setup_socks5() + if test_name == "ocsp": if sub_test_name: os.environ["OCSP_SERVER_TYPE"] = sub_test_name diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index 09f2925ef5..42574198b6 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -36,6 +36,11 @@ elif TEST_NAME == "ocsp": run_command(f"bash {DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh") +elif TEST_NAME == "socks5": + from run_socks5_proxy import stop_socks5_proxies + + stop_socks5_proxies([int(pid) for pid in os.environ.get("SOCKS5_PROXY_PIDS", "").split()]) + # Tear down atlas cluster if applicable. if TEST_NAME in ["aws_lambda", "search_index"]: run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh") diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index f078f2f305..1d9d2d4701 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -72,6 +72,36 @@ def variant_selects(variant, task): return False +def test_socks5_tasks_use_standard_setup_and_execution(): + tasks = generate_config.create_socks5_tasks() + assert {task.name for task in tasks} == { + "test-socks5-prose-noauth-nossl", + "test-socks5-prose-noauth-ssl", + } + assert {task_vars(task)["SSL"] for task in tasks} == {"nossl", "ssl"} + for task in tasks: + assert {"socks5", "pr"} <= set(task.tags) + assert [command.func for command in task.commands] == ["run server", "run tests"] + assert task.commands[0].vars == dict( + TEST_NAME="socks5", + VERSION=generate_config_utils.MAX_VERSION, + TOPOLOGY="replica_set", + AUTH="noauth", + SSL=task_vars(task)["SSL"], + ) + assert task.commands[1].vars == dict(TEST_NAME="socks5", COVERAGE="1") + + +def test_socks5_variant_selects_both_prose_tasks(): + (variant,) = generate_config.create_socks5_variants() + assert variant.name == "socks5-prose" + assert variant.display_name == "SOCKS5 Prose Tests" + assert "pr" in variant.tags + assert variant.expansions["UV_PYTHON"] == generate_config_utils.MIN_PYTHON + assert "SOCKS5_PROXIES_STARTED" not in variant.expansions + assert all(variant_selects(variant, task) for task in generate_config.create_socks5_tasks()) + + def test_pr_tasks_pin_max_version(): """Task-level pins: pr tasks pin MAX_VERSION (the "latest" nightly is unstable); the explicit 4.4 min-deps pin is the exception. Variant-merged diff --git a/.evergreen/scripts/test_socks5_proxy.py b/.evergreen/scripts/test_socks5_proxy.py new file mode 100644 index 0000000000..21e89488d6 --- /dev/null +++ b/.evergreen/scripts/test_socks5_proxy.py @@ -0,0 +1,169 @@ +"""Tests for SOCKS5 test setup, execution, and teardown.""" + +from __future__ import annotations + +import runpy +import signal +import subprocess +import sys +from pathlib import Path +from unittest import mock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent)) + +import run_socks5_proxy +import run_tests +import setup_tests +from utils import TEST_SUITE_MAP + + +@pytest.fixture +def proxy_setup(monkeypatch, tmp_path): + monkeypatch.setenv("MONGODB_URI", "mongodb://localhost:27017/") + monkeypatch.setattr(setup_tests, "HERE", tmp_path) + first = mock.Mock(pid=11) + second = mock.Mock(pid=22) + for process in (first, second): + process.poll.return_value = None + with ( + mock.patch.object(setup_tests, "check_drivers_tools"), + mock.patch.object(setup_tests, "write_env") as write_env, + mock.patch.object(setup_tests.socket, "socket") as socket_factory, + mock.patch.object(setup_tests.socket, "create_connection") as connect, + mock.patch.object(setup_tests.subprocess, "Popen", side_effect=[first, second]) as popen, + ): + yield first, second, popen, socket_factory, connect, write_env + + +def test_setup_starts_and_checks_both_proxies(proxy_setup): + first, second, popen, socket_factory, connect, write_env = proxy_setup + setup_tests.setup_socks5() + args = [call.args[0] for call in popen.call_args_list] + assert args == [ + [ + sys.executable, + str(setup_tests.HERE / "run_socks5_proxy.py"), + "--port", + "1080", + "--auth", + "username:p4ssw0rd", + ], + [sys.executable, str(setup_tests.HERE / "run_socks5_proxy.py"), "--port", "1081"], + ] + assert socket_factory.return_value.__enter__.return_value.bind.call_args_list == [ + mock.call(("127.0.0.1", 1080)), + mock.call(("127.0.0.1", 1081)), + ] + assert ( + socket_factory.return_value.__enter__.return_value.setsockopt.call_args_list + == [mock.call(setup_tests.socket.SOL_SOCKET, setup_tests.socket.SO_REUSEADDR, 1)] * 2 + ) + assert connect.call_args_list == [ + mock.call(("127.0.0.1", 1080), timeout=0.1), + mock.call(("127.0.0.1", 1081), timeout=0.1), + ] + assert write_env.call_args_list == [ + mock.call("SOCKS5_PROXY_PIDS", "11 22"), + mock.call("SOCKS5_PROXIES_STARTED"), + ] + first.terminate.assert_not_called() + second.terminate.assert_not_called() + + +def test_setup_cleans_up_after_second_proxy_fails(proxy_setup, monkeypatch): + first, second, _popen, _socket_factory, connect, write_env = proxy_setup + connect.side_effect = [mock.MagicMock(), ConnectionRefusedError("not ready")] + monkeypatch.setattr(setup_tests, "time", mock.Mock(monotonic=mock.Mock(side_effect=[0, 0, 11]))) + with pytest.raises(RuntimeError, match="proxy on port 1081 did not start"): + setup_tests.setup_socks5() + for process in (first, second): + process.terminate.assert_called_once_with() + process.wait.assert_called_once_with(timeout=10) + write_env.assert_not_called() + + +def test_setup_refuses_occupied_ports(proxy_setup): + _first, _second, popen, socket_factory, _connect, write_env = proxy_setup + socket_factory.return_value.__enter__.return_value.bind.side_effect = OSError("port in use") + with pytest.raises(OSError, match="port in use"): + setup_tests.setup_socks5() + popen.assert_not_called() + write_env.assert_not_called() + + +def test_teardown_stops_only_checkout_proxies(): + result = subprocess.CompletedProcess( + [], 0, stdout=f"{sys.executable} {run_socks5_proxy.__file__} --port 1080" + ) + with ( + mock.patch.object(run_socks5_proxy.subprocess, "run", return_value=result), + mock.patch.object(run_socks5_proxy.os, "kill") as kill, + ): + run_socks5_proxy.stop_socks5_proxies([11]) + kill.assert_called_once_with(11, signal.SIGTERM) + + +def test_teardown_refuses_reused_pid(): + result = subprocess.CompletedProcess([], 0, stdout="python unrelated.py") + with ( + mock.patch.object(run_socks5_proxy.subprocess, "run", return_value=result), + mock.patch.object(run_socks5_proxy.os, "kill") as kill, + pytest.raises(RuntimeError, match="Refusing to stop unrelated process"), + ): + run_socks5_proxy.stop_socks5_proxies([11]) + kill.assert_not_called() + + +@pytest.mark.parametrize("pid", [-1, 0]) +def test_teardown_rejects_invalid_pid(pid): + with ( + mock.patch.object(run_socks5_proxy.os, "kill") as kill, + pytest.raises(ValueError, match="Invalid SOCKS5 proxy PID"), + ): + run_socks5_proxy.stop_socks5_proxies([pid]) + kill.assert_not_called() + + +def test_teardown_script_uses_recorded_proxy_pids(monkeypatch): + monkeypatch.setenv("TEST_NAME", "socks5") + monkeypatch.setenv("SOCKS5_PROXY_PIDS", "11 22") + monkeypatch.setenv("COVERAGE", "") + with mock.patch.object(run_socks5_proxy, "stop_socks5_proxies") as stop: + runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) + stop.assert_called_once_with([11, 22]) + + +def test_runner_selects_sync_and_async_prose_tests(monkeypatch): + assert TEST_SUITE_MAP["socks5"] == "" + monkeypatch.setattr(run_tests, "TEST_NAME", "socks5") + monkeypatch.setattr(run_tests, "TEST_ARGS", ["-v"]) + monkeypatch.setattr(run_tests, "GREEN_FRAMEWORK", None) + monkeypatch.setattr(run_tests, "TEST_PERF", None) + monkeypatch.setattr(sys, "argv", ["run_tests.py"]) + monkeypatch.setenv("NO_EXT", "1") + monkeypatch.setenv("PYMONGOCRYPT_LIB", "") + monkeypatch.setenv("COVERAGE", "") + monkeypatch.setenv("DEBUG_LOG", "") + with ( + mock.patch.object(run_tests, "list_packages"), + mock.patch.object(run_tests.pytest, "main", return_value=0) as pytest_main, + ): + run_tests.run() + pytest_main.assert_called_once_with( + ["-v", "test/test_socks5.py", "test/asynchronous/test_socks5.py", "-k", "TestSocks5Prose"] + ) + + +@pytest.mark.parametrize( + ("uri", "expected"), + [ + ("mongodb://localhost/", "localhost:27017"), + ("mongodb://localhost:27018,localhost:27019/", "localhost:27018"), + ("mongodb://user:pass@localhost:27018/", "localhost:27018"), + ("mongodb://[::1]:27018,[::1]:27019/", "[::1]:27018"), + ], +) +def test_first_seed_mapping(uri, expected): + assert run_socks5_proxy._first_seed(uri) == expected diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 5b7da0f5a3..01d83bb3a8 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -49,6 +49,7 @@ class Distro: "ocsp": "ocsp", "perf": "perf", "sfp": "sfp", + "socks5": "", "numpy": "", } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e683cbc5d1..1cfbf602b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -458,6 +458,16 @@ wheels (CPython 3.12-3.14). then `just run-tests`. Set `FUZZ_MAX_TIME` (seconds, default 300) to change the per-target budget. +## SOCKS5 proxy tests + +With a local replica set running and `MONGODB_URI` set to its connection string, +run `just setup-tests socks5`, then `just run-tests`, and finally +`just teardown-tests`. Setup requires the drivers-evergreen-tools checkout and +starts authenticated and unauthenticated proxies on ports 1080 and 1081. The +runner exercises the synchronous and asynchronous SOCKS5 prose tests; teardown +stops the proxies. For a TLS-enabled replica set, use +`just setup-tests socks5 --ssl` instead. + ## Testing time-dependent operations - `test.utils_shared.delay` - One can trigger an arbitrarily long-running operation on the server using this delay utility diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 4b6f9c2e0a..a72c23187f 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -102,6 +102,8 @@ if TYPE_CHECKING: from pymongocrypt.mongocrypt import MongoCryptKmsContext + from pymongo.pool_options import _ProxyOptions + _IS_SYNC = False @@ -123,7 +125,7 @@ def __init__( key_vault_coll: AsyncCollection[_DocumentTypeArg], mongocryptd_client: Optional[AsyncMongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, - proxy_client: AsyncMongoClient[_DocumentTypeArg], + proxy_options: Optional[_ProxyOptions], ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -141,7 +143,7 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client - self._proxy_options = proxy_client.options.pool_options._proxy_options + self._proxy_options = proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) @@ -412,7 +414,7 @@ def _get_internal_client( key_vault_coll, # type:ignore[arg-type] mongocryptd_client, opts, - proxy_client=client, + proxy_options=client.options.pool_options._proxy_options, ) self._auto_encrypter = AsyncAutoEncrypter( io_callbacks, @@ -660,7 +662,11 @@ def __init__( ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( - None, key_vault_coll, None, opts, proxy_client=key_vault_client + None, + key_vault_coll, + None, + opts, + proxy_options=key_vault_client.options.pool_options._proxy_options, ) self._encryption = AsyncExplicitEncrypter( self._io_callbacks, diff --git a/pymongo/asynchronous/socks5_support.py b/pymongo/asynchronous/socks5_support.py new file mode 100644 index 0000000000..e891891eba --- /dev/null +++ b/pymongo/asynchronous/socks5_support.py @@ -0,0 +1,141 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Asynchronous SOCKS5 proxy connection establishment.""" + +from __future__ import annotations + +import socket +import time +from typing import TYPE_CHECKING, Optional + +from pymongo.pool_shared import _configure_tcp_socket, _new_tcp_socket +from pymongo.socks5_shared import ( + _SOCKS5_ATYP_DOMAIN, + _SOCKS5_AUTH_REPLY_LEN, + _SOCKS5_AUTH_USER_PASS, + _SOCKS5_CONNECT_REPLY_LEN, + _SOCKS5_DOMAIN_LENGTH_LEN, + _SOCKS5_GREETING_REPLY_LEN, + _async_proxy_connect, + _async_proxy_getaddrinfo, + _async_socks5_recv, + _async_socks5_sendall, + _proxy_deadline, + _proxy_socket_timeout, + _socks5_auth_request, + _socks5_bound_address_size, + _socks5_check_auth_reply, + _socks5_check_connect_reply, + _socks5_check_greeting_reply, + _socks5_greeting, + _socks5_request, + _Socks5Error, + _Socks5Timeout, +) + +if TYPE_CHECKING: + from pymongo.pool_options import PoolOptions, _ProxyOptions + from pymongo.typings import _Address + +_IS_SYNC = False + + +async def _async_socks5_read_exact( + sock: socket.socket, size: int, deadline: Optional[float] +) -> bytes: + data = bytearray() + while len(data) < size: + chunk = await _async_socks5_recv(sock, size - len(data), deadline) + if not chunk: + raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") + data.extend(chunk) + return bytes(data) + + +async def _async_socks5_handshake( + sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] +) -> None: + greeting, allowed = _socks5_greeting(proxy) + await _async_socks5_sendall(sock, greeting, deadline) + method = _socks5_check_greeting_reply( + await _async_socks5_read_exact(sock, _SOCKS5_GREETING_REPLY_LEN, deadline), allowed + ) + if method == _SOCKS5_AUTH_USER_PASS: + await _async_socks5_sendall(sock, _socks5_auth_request(proxy), deadline) + _socks5_check_auth_reply( + await _async_socks5_read_exact(sock, _SOCKS5_AUTH_REPLY_LEN, deadline) + ) + await _async_socks5_sendall(sock, _socks5_request(address), deadline) + atyp = _socks5_check_connect_reply( + await _async_socks5_read_exact(sock, _SOCKS5_CONNECT_REPLY_LEN, deadline) + ) + domain_length = ( + (await _async_socks5_read_exact(sock, _SOCKS5_DOMAIN_LENGTH_LEN, deadline))[0] + if atyp == _SOCKS5_ATYP_DOMAIN + else None + ) + bound_size = _socks5_bound_address_size(atyp, domain_length) + await _async_socks5_read_exact(sock, bound_size, deadline) + + +async def _async_create_proxy_connection( + address: _Address, options: PoolOptions, proxy: _ProxyOptions +) -> socket.socket: + deadline = _proxy_deadline(options) + family = socket.AF_INET + if socket.has_ipv6 and proxy.host != "localhost": + family = socket.AF_UNSPEC + try: + results = await _async_proxy_getaddrinfo(proxy, family, deadline) + except TimeoutError as exc: + raise _Socks5Timeout("timed out") from exc + except OSError as exc: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc + + err: Optional[OSError] = None + for af, socktype, proto, _, sockaddr in results: + sock = _new_tcp_socket(af, socktype, proto) + connected = False + try: + _configure_tcp_socket(sock, non_blocking=not _IS_SYNC) + await _async_proxy_connect(sock, sockaddr, deadline) + connected = True + await _async_socks5_handshake(sock, address, proxy, deadline) + sock.settimeout(_proxy_socket_timeout(options)) + return sock + except TimeoutError as exc: + sock.close() + if connected or (deadline is not None and deadline - time.monotonic() <= 0): + raise _Socks5Timeout("timed out") from exc + err = _Socks5Timeout("timed out") + err.__cause__ = exc + except OSError as exc: + sock.close() + if isinstance(exc, _Socks5Error): + raise + err = exc + # A connected proxy's handshake failure must not try a different + # resolved proxy address; the failure is specific to the endpoint. + if connected: + raise _Socks5Error(f"SOCKS5 handshake failed: {exc}") from exc + except BaseException: + sock.close() + raise + + if isinstance(err, socket.timeout): + raise err + if err is not None: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err + raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 9543b25a3b..31d214398a 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -20,7 +20,6 @@ import collections import contextlib import functools -import ipaddress import os import socket import ssl @@ -44,7 +43,6 @@ from pymongo.asynchronous.helpers import _getaddrinfo from pymongo.auth_shared import _AuthContext from pymongo.common import ( - _SOCKS5_MAX_STR_LEN, MAX_BSON_SIZE, MAX_MESSAGE_SIZE, MAX_WIRE_VERSION, @@ -70,9 +68,10 @@ NetworkingInterfaceBase, PyMongoProtocol, ) -from pymongo.pool_options import PoolOptions, _ProxyOptions +from pymongo.pool_options import PoolOptions from pymongo.server_api import _add_to_command from pymongo.socket_checker import SocketChecker +from pymongo.socks5_shared import _Socks5Error, _Socks5Timeout from pymongo.ssl_support import SSL_EOF_ERRORS, PYSSLError, SSLError, _has_sni SSLErrors = (PYSSLError, SSLError) @@ -82,35 +81,6 @@ from pymongo.pyopenssl_context import _sslConn from pymongo.typings import _Address, _AgnosticClientSession, _AgnosticMongoClient -_SOCKS5_VERSION = 0x05 -_SOCKS5_AUTH_VERSION = 0x01 -_SOCKS5_AUTH_NO_AUTH = 0x00 -_SOCKS5_AUTH_USER_PASS = 0x02 -_SOCKS5_AUTH_NO_ACCEPTABLE = 0xFF -_SOCKS5_AUTH_SUCCESS = 0x00 -_SOCKS5_RESERVED = 0x00 -_SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS = 1 -_SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS = 2 -_SOCKS5_CMD_CONNECT = 0x01 -_SOCKS5_REPLY_GENERAL_FAILURE = 0x01 -_SOCKS5_REPLY_RULESET_DENIED = 0x02 -_SOCKS5_REPLY_NETWORK_UNREACHABLE = 0x03 -_SOCKS5_REPLY_HOST_UNREACHABLE = 0x04 -_SOCKS5_REPLY_CONNECTION_REFUSED = 0x05 -_SOCKS5_REPLY_TTL_EXPIRED = 0x06 -_SOCKS5_REPLY_COMMAND_UNSUPPORTED = 0x07 -_SOCKS5_REPLY_ADDRESS_UNSUPPORTED = 0x08 -_SOCKS5_ATYP_IPV4 = 0x01 -_SOCKS5_ATYP_DOMAIN = 0x03 -_SOCKS5_ATYP_IPV6 = 0x04 -_SOCKS5_IPV4_BOUND_LEN = 6 -_SOCKS5_IPV6_BOUND_LEN = 18 -_SOCKS5_PORT_LEN = 2 -_SOCKS5_GREETING_REPLY_LEN = 2 -_SOCKS5_AUTH_REPLY_LEN = 2 -_SOCKS5_CONNECT_REPLY_LEN = 4 -_SOCKS5_DOMAIN_LENGTH_LEN = 1 - # Flags recording the counters a checkout has incremented, so a failed # checkout can restore them exactly once (PYTHON-6136). _UNDO_OPERATION_COUNT = 1 @@ -632,37 +602,6 @@ class PoolState: CLOSED = 3 -_SOCKS5_REPLY_REASONS = { - _SOCKS5_REPLY_GENERAL_FAILURE: "general SOCKS server failure", - _SOCKS5_REPLY_RULESET_DENIED: "connection not allowed by ruleset", - _SOCKS5_REPLY_NETWORK_UNREACHABLE: "network unreachable", - _SOCKS5_REPLY_HOST_UNREACHABLE: "host unreachable", - _SOCKS5_REPLY_CONNECTION_REFUSED: "connection refused", - _SOCKS5_REPLY_TTL_EXPIRED: "TTL expired", - _SOCKS5_REPLY_COMMAND_UNSUPPORTED: "command not supported", - _SOCKS5_REPLY_ADDRESS_UNSUPPORTED: "address type not supported", -} - - -class _Socks5Error(OSError): - """A SOCKS5 proxy connection or protocol failure.""" - - -class _Socks5Timeout(socket.timeout): - """A timeout while connecting through a SOCKS5 proxy.""" - - -def _proxy_deadline(options: PoolOptions) -> Optional[float]: - timeout = _csot.remaining() - if timeout is None: - timeout = options.connect_timeout - elif timeout <= 0: - raise _Socks5Timeout("timed out") - if timeout is None: - return None - return time.monotonic() + timeout - - def _new_tcp_socket(af: int, socktype: int, proto: int) -> socket.socket: # SOCK_CLOEXEC was added in CPython 3.2 and is not available on all # platforms or kernels. PEP 446 makes descriptors non-inheritable by @@ -683,308 +622,6 @@ def _configure_tcp_socket(sock: socket.socket, *, non_blocking: bool = False) -> sock.setblocking(False) -def _proxy_socket_timeout(options: PoolOptions) -> Optional[float]: - timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout - if timeout is not None and timeout <= 0: - raise _Socks5Timeout("timed out") - return timeout - - -def _socks5_bound_address_size(atyp: int, domain_length: Optional[int] = None) -> int: - if atyp == _SOCKS5_ATYP_IPV4: - return _SOCKS5_IPV4_BOUND_LEN - if atyp == _SOCKS5_ATYP_IPV6: - return _SOCKS5_IPV6_BOUND_LEN - if domain_length is None: - raise _Socks5Error("SOCKS5 connect failed: missing bound domain length") - return domain_length + _SOCKS5_PORT_LEN - - -def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: - if deadline is None: - return None - remaining = deadline - time.monotonic() - if remaining <= 0: - raise _Socks5Timeout("timed out") - return remaining - - -def _socks5_greeting(proxy: _ProxyOptions) -> tuple[bytes, tuple[int, ...]]: - if proxy.username is not None and proxy.password is not None: - return bytes( - ( - _SOCKS5_VERSION, - _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, - _SOCKS5_AUTH_NO_AUTH, - _SOCKS5_AUTH_USER_PASS, - ) - ), ( - _SOCKS5_AUTH_NO_AUTH, - _SOCKS5_AUTH_USER_PASS, - ) - return bytes( - (_SOCKS5_VERSION, _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, _SOCKS5_AUTH_NO_AUTH) - ), (_SOCKS5_AUTH_NO_AUTH,) - - -def _socks5_check_greeting_reply(data: bytes, allowed: tuple[int, ...]) -> int: - version, method = data - if version != _SOCKS5_VERSION: - raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") - if method not in allowed: - if method == _SOCKS5_AUTH_NO_ACCEPTABLE: - raise _Socks5Error( - "SOCKS5 initial handshake failed: no acceptable authentication methods" - ) - raise _Socks5Error( - f"SOCKS5 initial handshake failed: unexpected authentication method {method:#04x}" - ) - return method - - -def _socks5_auth_request(proxy: _ProxyOptions) -> bytes: - assert proxy.username is not None - assert proxy.password is not None - username = proxy.username.encode("utf-8") - password = proxy.password.encode("utf-8") - return ( - bytes((_SOCKS5_AUTH_VERSION, len(username))) + username + bytes((len(password),)) + password - ) - - -def _socks5_check_auth_reply(data: bytes) -> None: - auth_version, status = data - if auth_version != _SOCKS5_AUTH_VERSION or status != _SOCKS5_AUTH_SUCCESS: - raise _Socks5Error("SOCKS5 authentication failed") - - -def _socks5_request(address: _Address) -> bytes: - host, port = address - if port is None: - raise _Socks5Error("SOCKS5 connect failed: destination port is required") - try: - ip = ipaddress.ip_address(host.split("%", 1)[0]) - except ValueError: - try: - domain = host.encode("ascii") if host.isascii() else host.encode("idna") - except UnicodeError as exc: - raise _Socks5Error(f"SOCKS5 connect failed: invalid domain name: {exc}") from exc - if not 1 <= len(domain) <= _SOCKS5_MAX_STR_LEN: - raise _Socks5Error("SOCKS5 connect failed: invalid domain name length") from None - encoded_host = bytes((_SOCKS5_ATYP_DOMAIN, len(domain))) + domain - else: - encoded_host = ( - bytes((_SOCKS5_ATYP_IPV4 if ip.version == 4 else _SOCKS5_ATYP_IPV6,)) + ip.packed - ) - return ( - bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) - + encoded_host - + port.to_bytes(_SOCKS5_PORT_LEN, "big") - ) - - -def _socks5_check_connect_reply(data: bytes) -> int: - version, reply, reserved, atyp = data - if version != _SOCKS5_VERSION: - raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") - if reserved != _SOCKS5_RESERVED: - raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") - if reply: - reason = _SOCKS5_REPLY_REASONS.get(reply, f"unknown error {reply:#04x}") - raise _Socks5Error(f"SOCKS5 connect failed: {reason}") - if atyp not in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): - raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") - return atyp - - -def _socks5_read_exact(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: - data = bytearray() - while len(data) < size: - sock.settimeout(_proxy_remaining(deadline)) - chunk = sock.recv(size - len(data)) - if not chunk: - raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") - data.extend(chunk) - return bytes(data) - - -def _socks5_handshake( - sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] -) -> None: - greeting, allowed = _socks5_greeting(proxy) - sock.settimeout(_proxy_remaining(deadline)) - sock.sendall(greeting) - method = _socks5_check_greeting_reply( - _socks5_read_exact(sock, _SOCKS5_GREETING_REPLY_LEN, deadline), allowed - ) - if method == _SOCKS5_AUTH_USER_PASS: - sock.settimeout(_proxy_remaining(deadline)) - sock.sendall(_socks5_auth_request(proxy)) - _socks5_check_auth_reply(_socks5_read_exact(sock, _SOCKS5_AUTH_REPLY_LEN, deadline)) - - sock.settimeout(_proxy_remaining(deadline)) - sock.sendall(_socks5_request(address)) - atyp = _socks5_check_connect_reply( - _socks5_read_exact(sock, _SOCKS5_CONNECT_REPLY_LEN, deadline) - ) - domain_length = ( - _socks5_read_exact(sock, _SOCKS5_DOMAIN_LENGTH_LEN, deadline)[0] - if atyp == _SOCKS5_ATYP_DOMAIN - else None - ) - bound_size = _socks5_bound_address_size(atyp, domain_length) - _socks5_read_exact(sock, bound_size, deadline) - - -async def _async_socks5_read_exact( - sock: socket.socket, size: int, deadline: Optional[float] -) -> bytes: - data = bytearray() - loop = asyncio.get_running_loop() - while len(data) < size: - async with asyncio.timeout(_proxy_remaining(deadline)): - chunk = await loop.sock_recv(sock, size - len(data)) - if not chunk: - raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") - data.extend(chunk) - return bytes(data) - - -async def _async_socks5_handshake( - sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] -) -> None: - loop = asyncio.get_running_loop() - greeting, allowed = _socks5_greeting(proxy) - async with asyncio.timeout(_proxy_remaining(deadline)): - await loop.sock_sendall(sock, greeting) - method = _socks5_check_greeting_reply( - await _async_socks5_read_exact(sock, _SOCKS5_GREETING_REPLY_LEN, deadline), allowed - ) - if method == _SOCKS5_AUTH_USER_PASS: - async with asyncio.timeout(_proxy_remaining(deadline)): - await loop.sock_sendall(sock, _socks5_auth_request(proxy)) - _socks5_check_auth_reply( - await _async_socks5_read_exact(sock, _SOCKS5_AUTH_REPLY_LEN, deadline) - ) - async with asyncio.timeout(_proxy_remaining(deadline)): - await loop.sock_sendall(sock, _socks5_request(address)) - atyp = _socks5_check_connect_reply( - await _async_socks5_read_exact(sock, _SOCKS5_CONNECT_REPLY_LEN, deadline) - ) - domain_length = ( - (await _async_socks5_read_exact(sock, _SOCKS5_DOMAIN_LENGTH_LEN, deadline))[0] - if atyp == _SOCKS5_ATYP_DOMAIN - else None - ) - bound_size = _socks5_bound_address_size(atyp, domain_length) - await _async_socks5_read_exact(sock, bound_size, deadline) - - -async def _async_create_proxy_connection( - address: _Address, options: PoolOptions, proxy: _ProxyOptions -) -> socket.socket: - deadline = _proxy_deadline(options) - family = socket.AF_INET - if socket.has_ipv6 and proxy.host != "localhost": - family = socket.AF_UNSPEC - try: - async with asyncio.timeout(_proxy_remaining(deadline)): - results = await _getaddrinfo( - proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM - ) - except TimeoutError as exc: - raise _Socks5Timeout("timed out") from exc - except OSError as exc: - raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc - - err: Optional[OSError] = None - loop = asyncio.get_running_loop() - for af, socktype, proto, _, sockaddr in results: - sock = _new_tcp_socket(af, socktype, proto) - connected = False - try: - _configure_tcp_socket(sock, non_blocking=True) - async with asyncio.timeout(_proxy_remaining(deadline)): - await loop.sock_connect(sock, sockaddr) - connected = True - await _async_socks5_handshake(sock, address, proxy, deadline) - sock.settimeout(_proxy_socket_timeout(options)) - return sock - except TimeoutError as exc: - sock.close() - if connected or (deadline is not None and deadline - time.monotonic() <= 0): - raise _Socks5Timeout("timed out") from exc - err = _Socks5Timeout("timed out") - err.__cause__ = exc - except OSError as exc: - sock.close() - if isinstance(exc, _Socks5Error): - raise - err = exc - # A connected proxy's handshake failure must not try a different - # resolved proxy address; the failure is specific to the endpoint. - if connected: - raise _Socks5Error(f"SOCKS5 handshake failed: {exc}") from exc - except BaseException: - sock.close() - raise - - if isinstance(err, socket.timeout): - raise err - if err is not None: - raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err - raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") - - -def _create_proxy_connection( - address: _Address, options: PoolOptions, proxy: _ProxyOptions -) -> socket.socket: - deadline = _proxy_deadline(options) - family = socket.AF_INET - if socket.has_ipv6 and proxy.host != "localhost": - family = socket.AF_UNSPEC - try: - results = socket.getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) - except OSError as exc: - raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc - - err: Optional[OSError] = None - for af, socktype, proto, _, sockaddr in results: - sock = _new_tcp_socket(af, socktype, proto) - connected = False - try: - _configure_tcp_socket(sock) - sock.settimeout(_proxy_remaining(deadline)) - sock.connect(sockaddr) - connected = True - _socks5_handshake(sock, address, proxy, deadline) - sock.settimeout(_proxy_socket_timeout(options)) - return sock - except socket.timeout as exc: - sock.close() - if connected or (deadline is not None and deadline - time.monotonic() <= 0): - raise _Socks5Timeout("timed out") from exc - err = _Socks5Timeout("timed out") - err.__cause__ = exc - except OSError as exc: - sock.close() - if isinstance(exc, _Socks5Error): - raise - err = exc - if connected: - # Handshake failures are not address-resolution failures. - raise _Socks5Error(f"SOCKS5 handshake failed: {exc}") from exc - except BaseException: - sock.close() - raise - - if isinstance(err, socket.timeout): - raise err - if err is not None: - raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err - raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") - - async def _async_create_connection(address: _Address, options: PoolOptions) -> socket.socket: """Given (host, port) and PoolOptions, connect and return a raw socket object. @@ -995,6 +632,8 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s host, port = address if options._proxy_options is not None: + from pymongo.asynchronous.socks5_support import _async_create_proxy_connection + return await _async_create_proxy_connection(address, options, options._proxy_options) # Check if dealing with a unix domain socket @@ -1231,6 +870,8 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket host, port = address if options._proxy_options is not None: + from pymongo.synchronous.socks5_support import _create_proxy_connection + return _create_proxy_connection(address, options, options._proxy_options) # Check if dealing with a unix domain socket diff --git a/pymongo/socks5_shared.py b/pymongo/socks5_shared.py new file mode 100644 index 0000000000..6e0ea81d7f --- /dev/null +++ b/pymongo/socks5_shared.py @@ -0,0 +1,258 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SOCKS5 protocol utilities and socket helpers shared by both APIs.""" + +from __future__ import annotations + +import asyncio +import ipaddress +import socket +import time +from typing import TYPE_CHECKING, Optional + +from pymongo import _csot +from pymongo.asynchronous.helpers import _getaddrinfo +from pymongo.common import _SOCKS5_MAX_STR_LEN + +if TYPE_CHECKING: + from pymongo.pool_options import PoolOptions, _ProxyOptions + from pymongo.typings import _Address + +_SOCKS5_VERSION = 0x05 +_SOCKS5_AUTH_VERSION = 0x01 +_SOCKS5_AUTH_NO_AUTH = 0x00 +_SOCKS5_AUTH_USER_PASS = 0x02 +_SOCKS5_AUTH_NO_ACCEPTABLE = 0xFF +_SOCKS5_AUTH_SUCCESS = 0x00 +_SOCKS5_RESERVED = 0x00 +_SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS = 1 +_SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS = 2 +_SOCKS5_CMD_CONNECT = 0x01 +_SOCKS5_REPLY_GENERAL_FAILURE = 0x01 +_SOCKS5_REPLY_RULESET_DENIED = 0x02 +_SOCKS5_REPLY_NETWORK_UNREACHABLE = 0x03 +_SOCKS5_REPLY_HOST_UNREACHABLE = 0x04 +_SOCKS5_REPLY_CONNECTION_REFUSED = 0x05 +_SOCKS5_REPLY_TTL_EXPIRED = 0x06 +_SOCKS5_REPLY_COMMAND_UNSUPPORTED = 0x07 +_SOCKS5_REPLY_ADDRESS_UNSUPPORTED = 0x08 +_SOCKS5_ATYP_IPV4 = 0x01 +_SOCKS5_ATYP_DOMAIN = 0x03 +_SOCKS5_ATYP_IPV6 = 0x04 +_SOCKS5_IPV4_BOUND_LEN = 6 +_SOCKS5_IPV6_BOUND_LEN = 18 +_SOCKS5_PORT_LEN = 2 +_SOCKS5_GREETING_REPLY_LEN = 2 +_SOCKS5_AUTH_REPLY_LEN = 2 +_SOCKS5_CONNECT_REPLY_LEN = 4 +_SOCKS5_DOMAIN_LENGTH_LEN = 1 + +_SOCKS5_REPLY_REASONS = { + _SOCKS5_REPLY_GENERAL_FAILURE: "general SOCKS server failure", + _SOCKS5_REPLY_RULESET_DENIED: "connection not allowed by ruleset", + _SOCKS5_REPLY_NETWORK_UNREACHABLE: "network unreachable", + _SOCKS5_REPLY_HOST_UNREACHABLE: "host unreachable", + _SOCKS5_REPLY_CONNECTION_REFUSED: "connection refused", + _SOCKS5_REPLY_TTL_EXPIRED: "TTL expired", + _SOCKS5_REPLY_COMMAND_UNSUPPORTED: "command not supported", + _SOCKS5_REPLY_ADDRESS_UNSUPPORTED: "address type not supported", +} + +_ProxyAddress = tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes] +_ProxyAddressInfo = tuple[socket.AddressFamily, socket.SocketKind, int, str, _ProxyAddress] + + +class _Socks5Error(OSError): + """A SOCKS5 proxy connection or protocol failure.""" + + +class _Socks5Timeout(socket.timeout): + """A timeout while connecting through a SOCKS5 proxy.""" + + +def _proxy_deadline(options: PoolOptions) -> Optional[float]: + timeout = _csot.remaining() + if timeout is None: + timeout = options.connect_timeout + elif timeout <= 0: + raise _Socks5Timeout("timed out") + if timeout is None: + return None + return time.monotonic() + timeout + + +def _proxy_socket_timeout(options: PoolOptions) -> Optional[float]: + timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout + if timeout is not None and timeout <= 0: + raise _Socks5Timeout("timed out") + return timeout + + +def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: + if deadline is None: + return None + remaining = deadline - time.monotonic() + if remaining <= 0: + raise _Socks5Timeout("timed out") + return remaining + + +def _socks5_bound_address_size(atyp: int, domain_length: Optional[int] = None) -> int: + if atyp == _SOCKS5_ATYP_IPV4: + return _SOCKS5_IPV4_BOUND_LEN + if atyp == _SOCKS5_ATYP_IPV6: + return _SOCKS5_IPV6_BOUND_LEN + if domain_length is None: + raise _Socks5Error("SOCKS5 connect failed: missing bound domain length") + return domain_length + _SOCKS5_PORT_LEN + + +def _socks5_greeting(proxy: _ProxyOptions) -> tuple[bytes, tuple[int, ...]]: + if proxy.username is not None and proxy.password is not None: + return bytes( + ( + _SOCKS5_VERSION, + _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + ), ( + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + return bytes( + (_SOCKS5_VERSION, _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, _SOCKS5_AUTH_NO_AUTH) + ), (_SOCKS5_AUTH_NO_AUTH,) + + +def _socks5_check_greeting_reply(data: bytes, allowed: tuple[int, ...]) -> int: + version, method = data + if version != _SOCKS5_VERSION: + raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") + if method not in allowed: + if method == _SOCKS5_AUTH_NO_ACCEPTABLE: + raise _Socks5Error( + "SOCKS5 initial handshake failed: no acceptable authentication methods" + ) + raise _Socks5Error( + f"SOCKS5 initial handshake failed: unexpected authentication method {method:#04x}" + ) + return method + + +def _socks5_auth_request(proxy: _ProxyOptions) -> bytes: + assert proxy.username is not None + assert proxy.password is not None + username = proxy.username.encode("utf-8") + password = proxy.password.encode("utf-8") + return ( + bytes((_SOCKS5_AUTH_VERSION, len(username))) + username + bytes((len(password),)) + password + ) + + +def _socks5_check_auth_reply(data: bytes) -> None: + auth_version, status = data + if auth_version != _SOCKS5_AUTH_VERSION or status != _SOCKS5_AUTH_SUCCESS: + raise _Socks5Error("SOCKS5 authentication failed") + + +def _socks5_request(address: _Address) -> bytes: + host, port = address + if port is None: + raise _Socks5Error("SOCKS5 connect failed: destination port is required") + try: + ip = ipaddress.ip_address(host.split("%", 1)[0]) + except ValueError: + try: + domain = host.encode("ascii") if host.isascii() else host.encode("idna") + except UnicodeError as exc: + raise _Socks5Error(f"SOCKS5 connect failed: invalid domain name: {exc}") from exc + if not 1 <= len(domain) <= _SOCKS5_MAX_STR_LEN: + raise _Socks5Error("SOCKS5 connect failed: invalid domain name length") from None + encoded_host = bytes((_SOCKS5_ATYP_DOMAIN, len(domain))) + domain + else: + encoded_host = ( + bytes((_SOCKS5_ATYP_IPV4 if ip.version == 4 else _SOCKS5_ATYP_IPV6,)) + ip.packed + ) + return ( + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) + + encoded_host + + port.to_bytes(_SOCKS5_PORT_LEN, "big") + ) + + +def _socks5_check_connect_reply(data: bytes) -> int: + version, reply, reserved, atyp = data + if version != _SOCKS5_VERSION: + raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") + if reserved != _SOCKS5_RESERVED: + raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") + if reply: + reason = _SOCKS5_REPLY_REASONS.get(reply, f"unknown error {reply:#04x}") + raise _Socks5Error(f"SOCKS5 connect failed: {reason}") + if atyp not in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): + raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") + return atyp + + +def _socks5_sendall(sock: socket.socket, message: bytes, deadline: Optional[float]) -> None: + sock.settimeout(_proxy_remaining(deadline)) + sock.sendall(message) + + +async def _async_socks5_sendall( + sock: socket.socket, message: bytes, deadline: Optional[float] +) -> None: + loop = asyncio.get_running_loop() + async with asyncio.timeout(_proxy_remaining(deadline)): + await loop.sock_sendall(sock, message) + + +def _socks5_recv(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: + sock.settimeout(_proxy_remaining(deadline)) + return sock.recv(size) + + +async def _async_socks5_recv(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: + loop = asyncio.get_running_loop() + async with asyncio.timeout(_proxy_remaining(deadline)): + return await loop.sock_recv(sock, size) + + +def _proxy_getaddrinfo( + proxy: _ProxyOptions, family: int, deadline: Optional[float] +) -> list[_ProxyAddressInfo]: + _proxy_remaining(deadline) + return socket.getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) + + +async def _async_proxy_getaddrinfo( + proxy: _ProxyOptions, family: int, deadline: Optional[float] +) -> list[_ProxyAddressInfo]: + async with asyncio.timeout(_proxy_remaining(deadline)): + return await _getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) + + +def _proxy_connect(sock: socket.socket, address: _ProxyAddress, deadline: Optional[float]) -> None: + sock.settimeout(_proxy_remaining(deadline)) + sock.connect(address) + + +async def _async_proxy_connect( + sock: socket.socket, address: _ProxyAddress, deadline: Optional[float] +) -> None: + loop = asyncio.get_running_loop() + async with asyncio.timeout(_proxy_remaining(deadline)): + await loop.sock_connect(sock, address) diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index c2b41882d3..4c8cd8d6c0 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -101,6 +101,8 @@ if TYPE_CHECKING: from pymongocrypt.mongocrypt import MongoCryptKmsContext + from pymongo.pool_options import _ProxyOptions + _IS_SYNC = True @@ -122,7 +124,7 @@ def __init__( key_vault_coll: Collection[_DocumentTypeArg], mongocryptd_client: Optional[MongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, - proxy_client: MongoClient[_DocumentTypeArg], + proxy_options: Optional[_ProxyOptions], ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -140,7 +142,7 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client - self._proxy_options = proxy_client.options.pool_options._proxy_options + self._proxy_options = proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) @@ -409,7 +411,7 @@ def _get_internal_client( key_vault_coll, # type:ignore[arg-type] mongocryptd_client, opts, - proxy_client=client, + proxy_options=client.options.pool_options._proxy_options, ) self._auto_encrypter = AutoEncrypter( io_callbacks, @@ -653,7 +655,11 @@ def __init__( ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( - None, key_vault_coll, None, opts, proxy_client=key_vault_client + None, + key_vault_coll, + None, + opts, + proxy_options=key_vault_client.options.pool_options._proxy_options, ) self._encryption = ExplicitEncrypter( self._io_callbacks, diff --git a/pymongo/synchronous/socks5_support.py b/pymongo/synchronous/socks5_support.py new file mode 100644 index 0000000000..31da3d2375 --- /dev/null +++ b/pymongo/synchronous/socks5_support.py @@ -0,0 +1,137 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Synchronous SOCKS5 proxy connection establishment.""" + +from __future__ import annotations + +import socket +import time +from typing import TYPE_CHECKING, Optional + +from pymongo.pool_shared import _configure_tcp_socket, _new_tcp_socket +from pymongo.socks5_shared import ( + _SOCKS5_ATYP_DOMAIN, + _SOCKS5_AUTH_REPLY_LEN, + _SOCKS5_AUTH_USER_PASS, + _SOCKS5_CONNECT_REPLY_LEN, + _SOCKS5_DOMAIN_LENGTH_LEN, + _SOCKS5_GREETING_REPLY_LEN, + _proxy_connect, + _proxy_deadline, + _proxy_getaddrinfo, + _proxy_socket_timeout, + _socks5_auth_request, + _socks5_bound_address_size, + _socks5_check_auth_reply, + _socks5_check_connect_reply, + _socks5_check_greeting_reply, + _socks5_greeting, + _socks5_recv, + _socks5_request, + _socks5_sendall, + _Socks5Error, + _Socks5Timeout, +) + +if TYPE_CHECKING: + from pymongo.pool_options import PoolOptions, _ProxyOptions + from pymongo.typings import _Address + +_IS_SYNC = True + + +def _socks5_read_exact(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: + data = bytearray() + while len(data) < size: + chunk = _socks5_recv(sock, size - len(data), deadline) + if not chunk: + raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") + data.extend(chunk) + return bytes(data) + + +def _socks5_handshake( + sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] +) -> None: + greeting, allowed = _socks5_greeting(proxy) + _socks5_sendall(sock, greeting, deadline) + method = _socks5_check_greeting_reply( + _socks5_read_exact(sock, _SOCKS5_GREETING_REPLY_LEN, deadline), allowed + ) + if method == _SOCKS5_AUTH_USER_PASS: + _socks5_sendall(sock, _socks5_auth_request(proxy), deadline) + _socks5_check_auth_reply(_socks5_read_exact(sock, _SOCKS5_AUTH_REPLY_LEN, deadline)) + _socks5_sendall(sock, _socks5_request(address), deadline) + atyp = _socks5_check_connect_reply( + _socks5_read_exact(sock, _SOCKS5_CONNECT_REPLY_LEN, deadline) + ) + domain_length = ( + (_socks5_read_exact(sock, _SOCKS5_DOMAIN_LENGTH_LEN, deadline))[0] + if atyp == _SOCKS5_ATYP_DOMAIN + else None + ) + bound_size = _socks5_bound_address_size(atyp, domain_length) + _socks5_read_exact(sock, bound_size, deadline) + + +def _create_proxy_connection( + address: _Address, options: PoolOptions, proxy: _ProxyOptions +) -> socket.socket: + deadline = _proxy_deadline(options) + family = socket.AF_INET + if socket.has_ipv6 and proxy.host != "localhost": + family = socket.AF_UNSPEC + try: + results = _proxy_getaddrinfo(proxy, family, deadline) + except TimeoutError as exc: + raise _Socks5Timeout("timed out") from exc + except OSError as exc: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc + + err: Optional[OSError] = None + for af, socktype, proto, _, sockaddr in results: + sock = _new_tcp_socket(af, socktype, proto) + connected = False + try: + _configure_tcp_socket(sock, non_blocking=not _IS_SYNC) + _proxy_connect(sock, sockaddr, deadline) + connected = True + _socks5_handshake(sock, address, proxy, deadline) + sock.settimeout(_proxy_socket_timeout(options)) + return sock + except TimeoutError as exc: + sock.close() + if connected or (deadline is not None and deadline - time.monotonic() <= 0): + raise _Socks5Timeout("timed out") from exc + err = _Socks5Timeout("timed out") + err.__cause__ = exc + except OSError as exc: + sock.close() + if isinstance(exc, _Socks5Error): + raise + err = exc + # A connected proxy's handshake failure must not try a different + # resolved proxy address; the failure is specific to the endpoint. + if connected: + raise _Socks5Error(f"SOCKS5 handshake failed: {exc}") from exc + except BaseException: + sock.close() + raise + + if isinstance(err, socket.timeout): + raise err + if err is not None: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err + raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index 129ddefc66..3991a745af 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -37,7 +37,8 @@ from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions -from pymongo.pool_shared import _async_configured_socket, _async_create_connection, _Socks5Error +from pymongo.pool_shared import _async_configured_socket, _async_create_connection +from pymongo.socks5_shared import _Socks5Error from test.asynchronous import ( AsyncIntegrationTest, AsyncPyMongoTestCase, @@ -369,7 +370,7 @@ def close_tls_server() -> None: self.assertEqual(sni_names, ["mongodb.example"]) @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - async def test_kms_request_uses_client_proxy_options(self) -> None: + async def test_kms_request_uses_proxy_options(self) -> None: import pymongo.asynchronous.encryption as encryption_module from pymongo.asynchronous.encryption import _EncryptionIO from pymongo.asynchronous.mongo_client import AsyncMongoClient @@ -381,41 +382,42 @@ class FakeOpts: def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: return {"aws": object()} - client = AsyncMongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) - expected = _ProxyOptions("127.0.0.1") - io = _EncryptionIO( - client, - client.keyvault.datakeys, - None, - FakeOpts(), # type: ignore[arg-type] - proxy_client=client, - ) - connection = mock.MagicMock() - - async def connect(_address, options, _callback, _timeout): - self.assertEqual(options._proxy_options, expected) - return connection - - async def sendall(_connection, _message): - pass - + client = AsyncMongoClient("mongodb://localhost/", connect=False) try: - with ( - mock.patch.object(encryption_module, "_connect_kms", new=connect), - mock.patch.object(encryption_module, "async_socket_sendall", new=sendall), - ): - await io.kms_request( - SimpleNamespace( - endpoint="kms.example:443", - message=b"request", - kms_provider="aws", - usleep=0, - bytes_needed=0, + for expected in (None, _ProxyOptions("127.0.0.1")): + with self.subTest(proxy_options=expected): + io = _EncryptionIO( + client, + client.keyvault.datakeys, + None, + FakeOpts(), # type: ignore[arg-type] + proxy_options=expected, ) - ) + connection = mock.MagicMock() + + async def connect(_address, options, _callback, _timeout): + self.assertIs(options._proxy_options, expected) + return connection + + async def sendall(_connection, _message): + pass + + with ( + mock.patch.object(encryption_module, "_connect_kms", new=connect), + mock.patch.object(encryption_module, "async_socket_sendall", new=sendall), + ): + await io.kms_request( + SimpleNamespace( + endpoint="kms.example:443", + message=b"request", + kms_provider="aws", + usleep=0, + bytes_needed=0, + ) + ) + connection.close.assert_called_once_with() finally: await client.close() - connection.close.assert_called_once_with() async def test_proxy_settings_are_stored_in_pool_options(self) -> None: from pymongo.asynchronous.mongo_client import AsyncMongoClient @@ -498,7 +500,8 @@ async def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> Non async def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: from pymongo.asynchronous.pool import Pool from pymongo.errors import AutoReconnect, NetworkTimeout - from pymongo.pool_shared import _raise_connection_failure, _Socks5Timeout + from pymongo.pool_shared import _raise_connection_failure + from pymongo.socks5_shared import _Socks5Timeout pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) for cause in (_Socks5Error("boom"), _Socks5Timeout("timed out")): @@ -518,9 +521,9 @@ async def test_multi_address_proxy_fallback_on_connect_error(self) -> None: (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), ] target = ( - "pymongo.pool_shared.socket.getaddrinfo" + "pymongo.socks5_shared.socket.getaddrinfo" if _IS_SYNC - else "pymongo.pool_shared._getaddrinfo" + else "pymongo.socks5_shared._getaddrinfo" ) async def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: @@ -551,7 +554,7 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: kv_client.keyvault.datakeys, None, FakeOpts(), # type: ignore[arg-type] - proxy_client=parent_client, + proxy_options=parent_client.options.pool_options._proxy_options, ) self.assertEqual(io_bypass._proxy_options, _ProxyOptions("10.0.0.1")) @@ -560,7 +563,7 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: kv_client.keyvault.datakeys, None, FakeOpts(), # type: ignore[arg-type] - proxy_client=kv_client, + proxy_options=kv_client.options.pool_options._proxy_options, ) self.assertEqual(io_explicit._proxy_options, _ProxyOptions("10.0.0.2")) finally: @@ -585,54 +588,54 @@ async def test_proxy_port_validation(self) -> None: validate_proxy_port("proxyPort", True) async def test_proxy_deadline(self) -> None: - from pymongo.pool_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout + from pymongo.socks5_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) self.assertIsNone(_proxy_remaining(None)) with self.assertRaises(_Socks5Timeout): _proxy_remaining(0.0) with ( - mock.patch("pymongo.pool_shared._csot.remaining", return_value=0.0), + mock.patch("pymongo.socks5_shared._csot.remaining", return_value=0.0), self.assertRaises(_Socks5Timeout), ): _proxy_deadline(PoolOptions(connect_timeout=1)) async def test_proxy_socket_timeout_rejects_expired_deadline(self) -> None: - from pymongo.pool_shared import _proxy_socket_timeout, _Socks5Timeout + from pymongo.socks5_shared import _proxy_socket_timeout, _Socks5Timeout with ( - mock.patch("pymongo.pool_shared._csot.get_timeout", return_value=True), - mock.patch("pymongo.pool_shared._csot.remaining", return_value=-0.1), + mock.patch("pymongo.socks5_shared._csot.get_timeout", return_value=True), + mock.patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1), self.assertRaises(_Socks5Timeout), ): _proxy_socket_timeout(PoolOptions(connect_timeout=1)) async def test_socks5_request_rejects_long_domain(self) -> None: - from pymongo.pool_shared import _socks5_request + from pymongo.socks5_shared import _socks5_request with self.assertRaisesRegex(_Socks5Error, "invalid domain name length"): _socks5_request(("a" * 256, 27017)) async def test_socks5_request_rejects_invalid_domain(self) -> None: - from pymongo.pool_shared import _socks5_request + from pymongo.socks5_shared import _socks5_request with self.assertRaisesRegex(_Socks5Error, "invalid domain name"): _socks5_request((".münich.example", 27017)) async def test_socks5_request_checks_port_before_encoding_domain(self) -> None: - from pymongo.pool_shared import _socks5_request + from pymongo.socks5_shared import _socks5_request with self.assertRaisesRegex(_Socks5Error, "destination port is required"): _socks5_request(("a" * 256, None)) # type: ignore[arg-type] async def test_socks5_connect_reply_rejects_invalid_reserved_byte(self) -> None: - from pymongo.pool_shared import _socks5_check_connect_reply + from pymongo.socks5_shared import _socks5_check_connect_reply with self.assertRaisesRegex(_Socks5Error, "invalid reserved byte"): _socks5_check_connect_reply(b"\x05\x00\x01\x01") async def test_socks5_connect_reply_rejects_invalid_address_type(self) -> None: - from pymongo.pool_shared import _socks5_check_connect_reply + from pymongo.socks5_shared import _socks5_check_connect_reply with self.assertRaisesRegex(_Socks5Error, "invalid address type"): _socks5_check_connect_reply(b"\x05\x00\x00\x09") @@ -642,9 +645,9 @@ async def test_proxy_connection_error_paths(self) -> None: connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080) ) target = ( - "pymongo.pool_shared.socket.getaddrinfo" + "pymongo.socks5_shared.socket.getaddrinfo" if _IS_SYNC - else "pymongo.pool_shared._getaddrinfo" + else "pymongo.socks5_shared._getaddrinfo" ) def raise_gai(*_a: Any, **_kw: Any) -> Any: diff --git a/test/test_socks5.py b/test/test_socks5.py index f4989137e1..b83025c5d1 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -37,7 +37,8 @@ from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions -from pymongo.pool_shared import _configured_socket, _create_connection, _Socks5Error +from pymongo.pool_shared import _configured_socket, _create_connection +from pymongo.socks5_shared import _Socks5Error from test import ( IntegrationTest, PyMongoTestCase, @@ -369,7 +370,7 @@ def close_tls_server() -> None: self.assertEqual(sni_names, ["mongodb.example"]) @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - def test_kms_request_uses_client_proxy_options(self) -> None: + def test_kms_request_uses_proxy_options(self) -> None: import pymongo.synchronous.encryption as encryption_module from pymongo.synchronous.encryption import _EncryptionIO from pymongo.synchronous.mongo_client import MongoClient @@ -381,41 +382,42 @@ class FakeOpts: def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: return {"aws": object()} - client = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) - expected = _ProxyOptions("127.0.0.1") - io = _EncryptionIO( - client, - client.keyvault.datakeys, - None, - FakeOpts(), # type: ignore[arg-type] - proxy_client=client, - ) - connection = mock.MagicMock() - - def connect(_address, options, _callback, _timeout): - self.assertEqual(options._proxy_options, expected) - return connection - - def sendall(_connection, _message): - pass - + client = MongoClient("mongodb://localhost/", connect=False) try: - with ( - mock.patch.object(encryption_module, "_connect_kms", new=connect), - mock.patch.object(encryption_module, "sendall", new=sendall), - ): - io.kms_request( - SimpleNamespace( - endpoint="kms.example:443", - message=b"request", - kms_provider="aws", - usleep=0, - bytes_needed=0, + for expected in (None, _ProxyOptions("127.0.0.1")): + with self.subTest(proxy_options=expected): + io = _EncryptionIO( + client, + client.keyvault.datakeys, + None, + FakeOpts(), # type: ignore[arg-type] + proxy_options=expected, ) - ) + connection = mock.MagicMock() + + def connect(_address, options, _callback, _timeout): + self.assertIs(options._proxy_options, expected) + return connection + + def sendall(_connection, _message): + pass + + with ( + mock.patch.object(encryption_module, "_connect_kms", new=connect), + mock.patch.object(encryption_module, "sendall", new=sendall), + ): + io.kms_request( + SimpleNamespace( + endpoint="kms.example:443", + message=b"request", + kms_provider="aws", + usleep=0, + bytes_needed=0, + ) + ) + connection.close.assert_called_once_with() finally: client.close() - connection.close.assert_called_once_with() def test_proxy_settings_are_stored_in_pool_options(self) -> None: from pymongo.synchronous.mongo_client import MongoClient @@ -497,7 +499,8 @@ def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> None: def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: from pymongo.errors import AutoReconnect, NetworkTimeout - from pymongo.pool_shared import _raise_connection_failure, _Socks5Timeout + from pymongo.pool_shared import _raise_connection_failure + from pymongo.socks5_shared import _Socks5Timeout from pymongo.synchronous.pool import Pool pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) @@ -518,9 +521,9 @@ def test_multi_address_proxy_fallback_on_connect_error(self) -> None: (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), ] target = ( - "pymongo.pool_shared.socket.getaddrinfo" + "pymongo.socks5_shared.socket.getaddrinfo" if _IS_SYNC - else "pymongo.pool_shared._getaddrinfo" + else "pymongo.socks5_shared._getaddrinfo" ) def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: @@ -551,7 +554,7 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: kv_client.keyvault.datakeys, None, FakeOpts(), # type: ignore[arg-type] - proxy_client=parent_client, + proxy_options=parent_client.options.pool_options._proxy_options, ) self.assertEqual(io_bypass._proxy_options, _ProxyOptions("10.0.0.1")) @@ -560,7 +563,7 @@ def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: kv_client.keyvault.datakeys, None, FakeOpts(), # type: ignore[arg-type] - proxy_client=kv_client, + proxy_options=kv_client.options.pool_options._proxy_options, ) self.assertEqual(io_explicit._proxy_options, _ProxyOptions("10.0.0.2")) finally: @@ -585,54 +588,54 @@ def test_proxy_port_validation(self) -> None: validate_proxy_port("proxyPort", True) def test_proxy_deadline(self) -> None: - from pymongo.pool_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout + from pymongo.socks5_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) self.assertIsNone(_proxy_remaining(None)) with self.assertRaises(_Socks5Timeout): _proxy_remaining(0.0) with ( - mock.patch("pymongo.pool_shared._csot.remaining", return_value=0.0), + mock.patch("pymongo.socks5_shared._csot.remaining", return_value=0.0), self.assertRaises(_Socks5Timeout), ): _proxy_deadline(PoolOptions(connect_timeout=1)) def test_proxy_socket_timeout_rejects_expired_deadline(self) -> None: - from pymongo.pool_shared import _proxy_socket_timeout, _Socks5Timeout + from pymongo.socks5_shared import _proxy_socket_timeout, _Socks5Timeout with ( - mock.patch("pymongo.pool_shared._csot.get_timeout", return_value=True), - mock.patch("pymongo.pool_shared._csot.remaining", return_value=-0.1), + mock.patch("pymongo.socks5_shared._csot.get_timeout", return_value=True), + mock.patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1), self.assertRaises(_Socks5Timeout), ): _proxy_socket_timeout(PoolOptions(connect_timeout=1)) def test_socks5_request_rejects_long_domain(self) -> None: - from pymongo.pool_shared import _socks5_request + from pymongo.socks5_shared import _socks5_request with self.assertRaisesRegex(_Socks5Error, "invalid domain name length"): _socks5_request(("a" * 256, 27017)) def test_socks5_request_rejects_invalid_domain(self) -> None: - from pymongo.pool_shared import _socks5_request + from pymongo.socks5_shared import _socks5_request with self.assertRaisesRegex(_Socks5Error, "invalid domain name"): _socks5_request((".münich.example", 27017)) def test_socks5_request_checks_port_before_encoding_domain(self) -> None: - from pymongo.pool_shared import _socks5_request + from pymongo.socks5_shared import _socks5_request with self.assertRaisesRegex(_Socks5Error, "destination port is required"): _socks5_request(("a" * 256, None)) # type: ignore[arg-type] def test_socks5_connect_reply_rejects_invalid_reserved_byte(self) -> None: - from pymongo.pool_shared import _socks5_check_connect_reply + from pymongo.socks5_shared import _socks5_check_connect_reply with self.assertRaisesRegex(_Socks5Error, "invalid reserved byte"): _socks5_check_connect_reply(b"\x05\x00\x01\x01") def test_socks5_connect_reply_rejects_invalid_address_type(self) -> None: - from pymongo.pool_shared import _socks5_check_connect_reply + from pymongo.socks5_shared import _socks5_check_connect_reply with self.assertRaisesRegex(_Socks5Error, "invalid address type"): _socks5_check_connect_reply(b"\x05\x00\x00\x09") @@ -642,9 +645,9 @@ def test_proxy_connection_error_paths(self) -> None: connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080) ) target = ( - "pymongo.pool_shared.socket.getaddrinfo" + "pymongo.socks5_shared.socket.getaddrinfo" if _IS_SYNC - else "pymongo.pool_shared._getaddrinfo" + else "pymongo.socks5_shared._getaddrinfo" ) def raise_gai(*_a: Any, **_kw: Any) -> Any: diff --git a/tools/synchro.py b/tools/synchro.py index 4d2f865479..3c79c284e9 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -140,6 +140,13 @@ "async_joinall": "joinall", "async_simple_test_client": "simple_test_client", "_async_create_connection": "_create_connection", + "_async_create_proxy_connection": "_create_proxy_connection", + "_async_socks5_handshake": "_socks5_handshake", + "_async_socks5_read_exact": "_socks5_read_exact", + "_async_socks5_sendall": "_socks5_sendall", + "_async_socks5_recv": "_socks5_recv", + "_async_proxy_connect": "_proxy_connect", + "_async_proxy_getaddrinfo": "_proxy_getaddrinfo", "pymongo.asynchronous.srv_resolver._SrvResolver.get_hosts": "pymongo.synchronous.srv_resolver._SrvResolver.get_hosts", "dns.asyncresolver.resolve": "dns.resolver.resolve", "__aenter__": "__enter__", From fc92b78623ae2d46ad749e1177fb05afa002f9e6 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Fri, 6 Nov 2026 19:48:21 +0000 Subject: [PATCH 13/16] address review comments --- .evergreen/scripts/generate-config.sh | 2 +- .evergreen/scripts/run_socks5_proxy.py | 90 ---- .evergreen/scripts/setup_tests.py | 74 ++-- .evergreen/scripts/teardown_tests.py | 30 +- .evergreen/scripts/test_socks5_proxy.py | 106 ++--- doc/changelog.rst | 3 +- pymongo/_kms_connect_shared.py | 13 +- pymongo/asynchronous/_kms_connect.py | 3 +- pymongo/asynchronous/encryption.py | 18 +- pymongo/asynchronous/mongo_client.py | 4 + pymongo/asynchronous/socks5_support.py | 19 +- pymongo/common.py | 28 +- pymongo/encryption_options.py | 5 +- pymongo/pool_shared.py | 13 +- pymongo/socks5_shared.py | 17 +- pymongo/synchronous/_kms_connect.py | 3 +- pymongo/synchronous/encryption.py | 18 +- pymongo/synchronous/mongo_client.py | 4 + pymongo/synchronous/socks5_support.py | 19 +- pymongo/uri_parser_shared.py | 16 +- test/asynchronous/test_socks5.py | 534 +++++++++++++++++------- test/socks5_helpers.py | 87 ++-- test/test_kms_connect.py | 17 +- test/test_socks5.py | 532 ++++++++++++++++------- test/test_uri_parser.py | 31 ++ 25 files changed, 1119 insertions(+), 567 deletions(-) delete mode 100644 .evergreen/scripts/run_socks5_proxy.py diff --git a/.evergreen/scripts/generate-config.sh b/.evergreen/scripts/generate-config.sh index 4f46605901..669a3d3d59 100755 --- a/.evergreen/scripts/generate-config.sh +++ b/.evergreen/scripts/generate-config.sh @@ -4,6 +4,6 @@ set -eu # Assert the generation rules, then regenerate the config. -python -m pytest .evergreen/scripts/test_generate_config.py -m "" +python -m pytest .evergreen/scripts/test_generate_config.py .evergreen/scripts/test_socks5_proxy.py -m "" python .evergreen/scripts/generate_config.py diff --git a/.evergreen/scripts/run_socks5_proxy.py b/.evergreen/scripts/run_socks5_proxy.py deleted file mode 100644 index ca8e335c76..0000000000 --- a/.evergreen/scripts/run_socks5_proxy.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import annotations - -import argparse -import os -import runpy -import shlex -import signal -import subprocess -import sys -from pathlib import Path -from shutil import which -from urllib.parse import urlsplit - -from utils import LOGGER - - -def _first_seed(uri: str) -> str: - authority = urlsplit(uri).netloc.rsplit("@", 1)[-1] - seed = authority.split(",", 1)[0] - if seed.startswith("["): - closing_bracket = seed.find("]") - if closing_bracket < 0: - raise ValueError(f"Invalid IPv6 seed in MONGODB_URI: {seed}") - host = seed[1:closing_bracket] - suffix = seed[closing_bracket + 1 :] - if suffix and not suffix.startswith(":"): - raise ValueError(f"Invalid seed in MONGODB_URI: {seed}") - port = suffix[1:] if suffix else "27017" - else: - host, separator, port = seed.rpartition(":") - if not separator: - host, port = seed, "27017" - if not host or not port.isdecimal(): - raise ValueError(f"Invalid seed in MONGODB_URI: {seed}") - return f"[{host}]:{port}" if ":" in host else f"{host}:{port}" - - -def stop_socks5_proxies(pids: list[int]) -> None: - """Stop the proxy processes started for this checkout. - - :param pids: Process IDs recorded by the test setup. - :raises ValueError: If a process ID is not positive. - :raises RuntimeError: If ``ps`` is missing or a process is unrelated. - """ - ps = which("ps") - if ps is None: - raise RuntimeError("Cannot locate ps to verify SOCKS5 proxy process ownership") - script = str(Path(__file__).absolute()) - for pid in pids: - if pid <= 0: - raise ValueError(f"Invalid SOCKS5 proxy PID: {pid}") - result = subprocess.run( # noqa: S603 - [ps, "-ww", "-p", str(pid), "-o", "args="], - capture_output=True, - text=True, - check=False, - ) - if result.returncode == 1: - LOGGER.info("SOCKS5 proxy PID %s already exited", pid) - continue - result.check_returncode() - if script not in shlex.split(result.stdout): - raise RuntimeError(f"Refusing to stop unrelated process with PID {pid}") - try: - os.kill(pid, signal.SIGTERM) - except ProcessLookupError: - LOGGER.info("SOCKS5 proxy PID %s already exited", pid) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--port", type=int, required=True) - parser.add_argument("--auth") - args = parser.parse_args() - - script = Path(os.environ["DRIVERS_TOOLS"]) / ".evergreen" / "socks5srv.py" - sys.argv = [ - str(script), - "--port", - str(args.port), - "--map", - f"localhost:12345 to {_first_seed(os.environ['MONGODB_URI'])}", - ] - if args.auth: - sys.argv.extend(("--auth", args.auth)) - runpy.run_path(str(script), run_name="__main__") - - -if __name__ == "__main__": - main() diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 25c8e0b2b9..ddfdd45098 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -12,6 +12,7 @@ import zipfile from pathlib import Path from urllib import request +from urllib.parse import urlsplit from utils import ( DRIVERS_TOOLS, @@ -55,6 +56,12 @@ "pyopenssl": "ocsp", } +SOCKS5_AUTH_PORT = 1080 +SOCKS5_NOAUTH_PORT = 1081 +SOCKS5_MAPPED_PORT = 12345 +SOCKS5_PROXY_STARTUP_TIMEOUT = 10 +SOCKS5_PROXY_POLL_INTERVAL = 0.1 + # Map the test name to test group. GROUP_MAP = dict(fuzz="fuzz", mockupdb="mockupdb", perf="perf") @@ -180,56 +187,69 @@ def get_secrets(name: str) -> dict[str, str]: return load_config_from_file(secrets_dir / "secrets-export.sh") +def _first_seed(uri: str) -> str: + authority = urlsplit(uri).netloc.rsplit("@", 1)[-1] + seed = urlsplit(f"mongodb://{authority.split(',', 1)[0]}") + host = seed.hostname + if not host: + raise ValueError(f"Invalid seed in MONGODB_URI: {authority}") + port = seed.port if seed.port is not None else 27017 + return f"[{host}]:{port}" if ":" in host else f"{host}:{port}" + + +def _wait_for_socks5_proxy(process: subprocess.Popen[bytes], port: int, log_path: Path) -> None: + """Wait for a SOCKS5 proxy to accept connections. + + :param process: The proxy process to monitor. + :param port: The local port on which the proxy should listen. + :param log_path: The proxy's log file, included in errors. + :raises RuntimeError: If the process exits or does not become ready in time. + """ + deadline = time.monotonic() + SOCKS5_PROXY_STARTUP_TIMEOUT + while True: + if process.poll() is not None: + raise RuntimeError(f"SOCKS5 proxy on port {port} exited; see {log_path}") + try: + with socket.create_connection(("127.0.0.1", port), timeout=SOCKS5_PROXY_POLL_INTERVAL): + return + except OSError as exc: + if time.monotonic() >= deadline: + raise RuntimeError( + f"SOCKS5 proxy on port {port} did not start; see {log_path}" + ) from exc + time.sleep(SOCKS5_PROXY_POLL_INTERVAL) + + def setup_socks5() -> None: """Start and verify the authenticated and unauthenticated test proxies. - :raises ValueError: If ``MONGODB_URI`` is unset. - :raises OSError: If a proxy port is already in use. + :raises ValueError: If ``MONGODB_URI`` is unset or invalid. :raises RuntimeError: If the tools checkout is missing or a proxy fails to start. """ check_drivers_tools() if not os.environ.get("MONGODB_URI"): raise ValueError("MONGODB_URI must point to the replica set for SOCKS5 tests") - ports = (1080, 1081) - for port in ports: - with socket.socket() as sock: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("127.0.0.1", port)) - - env = os.environ.copy() - env["DRIVERS_TOOLS"] = DRIVERS_TOOLS + script = (Path(DRIVERS_TOOLS) / ".evergreen" / "socks5srv.py").absolute() + mapping = f"localhost:{SOCKS5_MAPPED_PORT} to {_first_seed(os.environ['MONGODB_URI'])}" processes: list[subprocess.Popen[bytes]] = [] started = False try: - for port in ports: - args = [sys.executable, str(HERE / "run_socks5_proxy.py"), "--port", str(port)] - if port == 1080: + for port in (SOCKS5_AUTH_PORT, SOCKS5_NOAUTH_PORT): + args = [sys.executable, str(script), "--port", str(port), "--map", mapping] + if port == SOCKS5_AUTH_PORT: args.extend(("--auth", "username:p4ssw0rd")) log_path = HERE / f"socks5-proxy-{port}.log" LOGGER.info("Starting SOCKS5 proxy on port %s; log: %s", port, log_path) with log_path.open("wb") as log: process = subprocess.Popen( # noqa: S603 args, - env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, ) processes.append(process) - deadline = time.monotonic() + 10 - while True: - if process.poll() is not None: - raise RuntimeError(f"SOCKS5 proxy on port {port} exited; see {log_path}") - try: - with socket.create_connection(("127.0.0.1", port), timeout=0.1): - break - except OSError as exc: - if time.monotonic() >= deadline: - raise RuntimeError( - f"SOCKS5 proxy on port {port} did not start; see {log_path}" - ) from exc - time.sleep(0.1) + _wait_for_socks5_proxy(process, port, log_path) write_env("SOCKS5_PROXY_PIDS", " ".join(str(process.pid) for process in processes)) write_env("SOCKS5_PROXIES_STARTED") started = True diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index 42574198b6..d564dcca2d 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -1,7 +1,10 @@ from __future__ import annotations import os +import shlex import shutil +import signal +import subprocess import sys from pathlib import Path @@ -37,9 +40,30 @@ run_command(f"bash {DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh") elif TEST_NAME == "socks5": - from run_socks5_proxy import stop_socks5_proxies - - stop_socks5_proxies([int(pid) for pid in os.environ.get("SOCKS5_PROXY_PIDS", "").split()]) + ps = shutil.which("ps") + if ps is None: + raise RuntimeError("Cannot locate ps to verify SOCKS5 proxy process ownership") + script = str((Path(DRIVERS_TOOLS) / ".evergreen" / "socks5srv.py").absolute()) + for value in os.environ.get("SOCKS5_PROXY_PIDS", "").split(): + pid = int(value) + if pid <= 0: + raise ValueError(f"Invalid SOCKS5 proxy PID: {pid}") + result = subprocess.run( # noqa: S603 + [ps, "-ww", "-p", str(pid), "-o", "args="], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 1: + LOGGER.info("SOCKS5 proxy PID %s already exited", pid) + continue + result.check_returncode() + if script not in shlex.split(result.stdout): + raise RuntimeError(f"Refusing to stop unrelated process with PID {pid}") + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + LOGGER.info("SOCKS5 proxy PID %s already exited", pid) # Tear down atlas cluster if applicable. if TEST_NAME in ["aws_lambda", "search_index"]: diff --git a/.evergreen/scripts/test_socks5_proxy.py b/.evergreen/scripts/test_socks5_proxy.py index 21e89488d6..9a1e36b039 100644 --- a/.evergreen/scripts/test_socks5_proxy.py +++ b/.evergreen/scripts/test_socks5_proxy.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import runpy import signal import subprocess @@ -13,7 +14,6 @@ sys.path.insert(0, str(Path(__file__).parent)) -import run_socks5_proxy import run_tests import setup_tests from utils import TEST_SUITE_MAP @@ -30,36 +30,37 @@ def proxy_setup(monkeypatch, tmp_path): with ( mock.patch.object(setup_tests, "check_drivers_tools"), mock.patch.object(setup_tests, "write_env") as write_env, - mock.patch.object(setup_tests.socket, "socket") as socket_factory, mock.patch.object(setup_tests.socket, "create_connection") as connect, mock.patch.object(setup_tests.subprocess, "Popen", side_effect=[first, second]) as popen, ): - yield first, second, popen, socket_factory, connect, write_env + yield first, second, popen, connect, write_env def test_setup_starts_and_checks_both_proxies(proxy_setup): - first, second, popen, socket_factory, connect, write_env = proxy_setup + first, second, popen, connect, write_env = proxy_setup setup_tests.setup_socks5() + script = str((Path(setup_tests.DRIVERS_TOOLS) / ".evergreen" / "socks5srv.py").absolute()) args = [call.args[0] for call in popen.call_args_list] assert args == [ [ sys.executable, - str(setup_tests.HERE / "run_socks5_proxy.py"), + script, "--port", "1080", + "--map", + "localhost:12345 to localhost:27017", "--auth", "username:p4ssw0rd", ], - [sys.executable, str(setup_tests.HERE / "run_socks5_proxy.py"), "--port", "1081"], - ] - assert socket_factory.return_value.__enter__.return_value.bind.call_args_list == [ - mock.call(("127.0.0.1", 1080)), - mock.call(("127.0.0.1", 1081)), + [ + sys.executable, + script, + "--port", + "1081", + "--map", + "localhost:12345 to localhost:27017", + ], ] - assert ( - socket_factory.return_value.__enter__.return_value.setsockopt.call_args_list - == [mock.call(setup_tests.socket.SOL_SOCKET, setup_tests.socket.SO_REUSEADDR, 1)] * 2 - ) assert connect.call_args_list == [ mock.call(("127.0.0.1", 1080), timeout=0.1), mock.call(("127.0.0.1", 1081), timeout=0.1), @@ -73,7 +74,7 @@ def test_setup_starts_and_checks_both_proxies(proxy_setup): def test_setup_cleans_up_after_second_proxy_fails(proxy_setup, monkeypatch): - first, second, _popen, _socket_factory, connect, write_env = proxy_setup + first, second, _popen, connect, write_env = proxy_setup connect.side_effect = [mock.MagicMock(), ConnectionRefusedError("not ready")] monkeypatch.setattr(setup_tests, "time", mock.Mock(monotonic=mock.Mock(side_effect=[0, 0, 11]))) with pytest.raises(RuntimeError, match="proxy on port 1081 did not start"): @@ -84,55 +85,60 @@ def test_setup_cleans_up_after_second_proxy_fails(proxy_setup, monkeypatch): write_env.assert_not_called() -def test_setup_refuses_occupied_ports(proxy_setup): - _first, _second, popen, socket_factory, _connect, write_env = proxy_setup - socket_factory.return_value.__enter__.return_value.bind.side_effect = OSError("port in use") - with pytest.raises(OSError, match="port in use"): +def test_setup_rejects_exited_proxy(proxy_setup): + first, _second, popen, _connect, write_env = proxy_setup + first.poll.return_value = 1 + with pytest.raises(RuntimeError, match="proxy on port 1080 exited"): setup_tests.setup_socks5() - popen.assert_not_called() + popen.assert_called_once() + first.terminate.assert_not_called() + first.wait.assert_called_once_with(timeout=10) write_env.assert_not_called() -def test_teardown_stops_only_checkout_proxies(): - result = subprocess.CompletedProcess( - [], 0, stdout=f"{sys.executable} {run_socks5_proxy.__file__} --port 1080" - ) +@pytest.fixture +def proxy_teardown(monkeypatch): + monkeypatch.setenv("TEST_NAME", "socks5") + monkeypatch.setenv("SOCKS5_PROXY_PIDS", "11 22") + monkeypatch.setenv("COVERAGE", "") + script = str((Path(setup_tests.DRIVERS_TOOLS) / ".evergreen" / "socks5srv.py").absolute()) + result = subprocess.CompletedProcess([], 0, stdout=f"{sys.executable} {script} --port 1080") with ( - mock.patch.object(run_socks5_proxy.subprocess, "run", return_value=result), - mock.patch.object(run_socks5_proxy.os, "kill") as kill, + mock.patch("shutil.which", return_value="/usr/bin/ps"), + mock.patch.object(subprocess, "run", return_value=result) as run, + mock.patch.object(os, "kill") as kill, ): - run_socks5_proxy.stop_socks5_proxies([11]) - kill.assert_called_once_with(11, signal.SIGTERM) + yield result, run, kill -def test_teardown_refuses_reused_pid(): - result = subprocess.CompletedProcess([], 0, stdout="python unrelated.py") - with ( - mock.patch.object(run_socks5_proxy.subprocess, "run", return_value=result), - mock.patch.object(run_socks5_proxy.os, "kill") as kill, - pytest.raises(RuntimeError, match="Refusing to stop unrelated process"), - ): - run_socks5_proxy.stop_socks5_proxies([11]) +def test_teardown_stops_recorded_proxies(proxy_teardown): + _result, _run, kill = proxy_teardown + runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) + assert kill.call_args_list == [mock.call(11, signal.SIGTERM), mock.call(22, signal.SIGTERM)] + + +def test_teardown_refuses_reused_pid(proxy_teardown): + result, _run, kill = proxy_teardown + result.stdout = "python unrelated.py" + with pytest.raises(RuntimeError, match="Refusing to stop unrelated process"): + runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) kill.assert_not_called() @pytest.mark.parametrize("pid", [-1, 0]) -def test_teardown_rejects_invalid_pid(pid): - with ( - mock.patch.object(run_socks5_proxy.os, "kill") as kill, - pytest.raises(ValueError, match="Invalid SOCKS5 proxy PID"), - ): - run_socks5_proxy.stop_socks5_proxies([pid]) +def test_teardown_rejects_invalid_pid(pid, proxy_teardown, monkeypatch): + _result, _run, kill = proxy_teardown + monkeypatch.setenv("SOCKS5_PROXY_PIDS", str(pid)) + with pytest.raises(ValueError, match="Invalid SOCKS5 proxy PID"): + runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) kill.assert_not_called() -def test_teardown_script_uses_recorded_proxy_pids(monkeypatch): - monkeypatch.setenv("TEST_NAME", "socks5") - monkeypatch.setenv("SOCKS5_PROXY_PIDS", "11 22") - monkeypatch.setenv("COVERAGE", "") - with mock.patch.object(run_socks5_proxy, "stop_socks5_proxies") as stop: - runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) - stop.assert_called_once_with([11, 22]) +def test_teardown_accepts_exited_proxies(proxy_teardown): + result, _run, kill = proxy_teardown + result.returncode = 1 + runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) + kill.assert_not_called() def test_runner_selects_sync_and_async_prose_tests(monkeypatch): @@ -166,4 +172,4 @@ def test_runner_selects_sync_and_async_prose_tests(monkeypatch): ], ) def test_first_seed_mapping(uri, expected): - assert run_socks5_proxy._first_seed(uri) == expected + assert setup_tests._first_seed(uri) == expected diff --git a/doc/changelog.rst b/doc/changelog.rst index c4ad313ca5..aa8553f650 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -9,7 +9,8 @@ Changes in Version 4.20.0 (2026/XX/XX) ``proxyPort``, ``proxyUsername``, and ``proxyPassword`` URI and keyword options on :class:`~pymongo.synchronous.mongo_client.MongoClient` and :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` - (`PYTHON-2943`_). + (`PYTHON-2943`_). SOCKS5 proxy options cannot be combined with + ``kms_connect_callback``. - The client handshake metadata now reports a detected coding agent in ``client.env.agent`` (`PYTHON-5929`_). diff --git a/pymongo/_kms_connect_shared.py b/pymongo/_kms_connect_shared.py index 47665181ee..634b4fe56e 100644 --- a/pymongo/_kms_connect_shared.py +++ b/pymongo/_kms_connect_shared.py @@ -32,10 +32,13 @@ import urllib.parse from collections.abc import Awaitable, Mapping from dataclasses import dataclass -from typing import Any, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable, Optional from pymongo.errors import ConfigurationError +if TYPE_CHECKING: + from pymongo.pool_options import _ProxyOptions + @dataclass(frozen=True) class KMSConnectContext: @@ -71,6 +74,14 @@ class KMSConnectContext: KMSConnectCallback = Callable[[KMSConnectContext], socket.socket] +def _check_kms_proxy_options( + proxy_options: Optional[_ProxyOptions], + kms_connect_callback: Optional[Callable[[KMSConnectContext], Any]], +) -> None: + if proxy_options is not None and kms_connect_callback is not None: + raise ConfigurationError("Cannot specify kms_connect_callback with proxyHost") + + def _close_rejected_kms_socket(obj: Any) -> None: """Close a callback return value that failed validation, best effort. diff --git a/pymongo/asynchronous/_kms_connect.py b/pymongo/asynchronous/_kms_connect.py index ab74919beb..8b516c472d 100644 --- a/pymongo/asynchronous/_kms_connect.py +++ b/pymongo/asynchronous/_kms_connect.py @@ -63,7 +63,8 @@ async def _connect_kms( """Connect to a KMS host and perform the TLS handshake over the socket. Uses ``kms_connect_callback`` when one is provided, otherwise connects - directly, and always verifies against ``address`` (the KMS host). + using the configured SOCKS5 proxy or directly. Always verifies against + ``address`` (the KMS host). """ if kms_connect_callback is None: try: diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index a72c23187f..3eee5c779e 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -53,7 +53,7 @@ from bson.codec_options import CodecOptions from bson.raw_bson import DEFAULT_RAW_BSON_OPTIONS, RawBSONDocument, _inflate_bson from pymongo import _csot, _op_id -from pymongo._kms_connect_shared import AsyncKMSConnectCallback +from pymongo._kms_connect_shared import AsyncKMSConnectCallback, _check_kms_proxy_options from pymongo.asynchronous._kms_connect import _KMS_CONNECT_TIMEOUT, _connect_kms from pymongo.asynchronous.collection import AsyncCollection from pymongo.asynchronous.cursor import AsyncCursor @@ -365,6 +365,9 @@ def __init__(self, client: AsyncMongoClient[_DocumentTypeArg], opts: AutoEncrypt :param client: The encrypted AsyncMongoClient. :param opts: The encrypted client's :class:`AutoEncryptionOpts`. """ + _check_kms_proxy_options( + client.options.pool_options._proxy_options, opts._kms_connect_callback + ) if opts._schema_map is None: schema_map = None else: @@ -578,7 +581,8 @@ def __init__( collection. Data keys are protected with encryption by a KMS provider. :param key_vault_client: An AsyncMongoClient connected to a MongoDB cluster - containing the `key_vault_namespace` collection. + containing the `key_vault_namespace` collection. Its SOCKS5 proxy + options are also used for KMS requests. :param codec_options: An instance of :class:`~bson.codec_options.CodecOptions` to use when encoding a value for encryption and decoding the decrypted BSON value. This @@ -610,8 +614,11 @@ def __init__( deadline and closes any socket the callback yields later. For an ordinary HTTP proxy, pass :class:`~pymongo.encryption_options.AsyncHTTPProxyKMSConnect`. - Defaults to ``None``, meaning the driver connects to KMS hosts - directly. + Defaults to ``None``, meaning the driver uses the SOCKS5 proxy + configured on ``key_vault_client``, or connects directly if none + is configured. Cannot be combined with ``proxyHost`` on + ``key_vault_client``; doing so raises + :class:`~pymongo.errors.ConfigurationError`. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. @@ -645,6 +652,9 @@ def __init__( f"AsyncMongoClient required but given {type(key_vault_client).__name__}" ) + _check_kms_proxy_options( + key_vault_client.options.pool_options._proxy_options, kms_connect_callback + ) self._kms_providers = kms_providers self._key_vault_namespace = key_vault_namespace self._key_vault_client = key_vault_client diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index a115d1f66c..35f4e2985b 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -336,6 +336,10 @@ def __init__( Defaults to ``20000`` (20 seconds). - `proxyHost` (string): The hostname or IP address of a SOCKS5 proxy. IPv6 literals may be enclosed in brackets. Defaults to no proxy. + Routes MongoDB and automatic encryption KMS connections through + the proxy. Cannot be combined with ``kms_connect_callback`` in + ``auto_encryption_opts``; doing so raises + :class:`~pymongo.errors.ConfigurationError`. - `proxyPort` (integer): The TCP port of the SOCKS5 proxy. Defaults to ``1080`` when ``proxyHost`` is specified. - `proxyUsername` (string): The username for SOCKS5 username/password diff --git a/pymongo/asynchronous/socks5_support.py b/pymongo/asynchronous/socks5_support.py index e891891eba..4c6038df7e 100644 --- a/pymongo/asynchronous/socks5_support.py +++ b/pymongo/asynchronous/socks5_support.py @@ -17,7 +17,6 @@ from __future__ import annotations import socket -import time from typing import TYPE_CHECKING, Optional from pymongo.pool_shared import _configure_tcp_socket, _new_tcp_socket @@ -33,6 +32,7 @@ _async_socks5_recv, _async_socks5_sendall, _proxy_deadline, + _proxy_remaining, _proxy_socket_timeout, _socks5_auth_request, _socks5_bound_address_size, @@ -100,7 +100,7 @@ async def _async_create_proxy_connection( try: results = await _async_proxy_getaddrinfo(proxy, family, deadline) except TimeoutError as exc: - raise _Socks5Timeout("timed out") from exc + raise _Socks5Timeout("SOCKS5 proxy timed out") from exc except OSError as exc: raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc @@ -117,14 +117,19 @@ async def _async_create_proxy_connection( return sock except TimeoutError as exc: sock.close() - if connected or (deadline is not None and deadline - time.monotonic() <= 0): - raise _Socks5Timeout("timed out") from exc - err = _Socks5Timeout("timed out") + if connected: + raise _Socks5Timeout("SOCKS5 proxy timed out") from exc + try: + _proxy_remaining(deadline) + except _Socks5Timeout as timeout: + raise timeout from exc + err = _Socks5Timeout("SOCKS5 proxy timed out") err.__cause__ = exc + except _Socks5Error: + sock.close() + raise except OSError as exc: sock.close() - if isinstance(exc, _Socks5Error): - raise err = exc # A connected proxy's handshake failure must not try a different # resolved proxy address; the failure is specific to the endpoint. diff --git a/pymongo/common.py b/pymongo/common.py index 9e4ad2368d..d8b93d5d87 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -236,7 +236,11 @@ def validate_non_negative_integer(option: str, value: Any) -> int: def validate_proxy_host(option: str, value: Any) -> Optional[str]: - """Validate and normalize a SOCKS5 proxy host.""" + """Validate and normalize a SOCKS5 proxy host. + + :raises TypeError: If ``value`` is not a string or ``None``. + :raises ValueError: If ``value`` is an invalid host. + """ if value is None: return None host = validate_string(option, value) @@ -264,7 +268,11 @@ def validate_proxy_host(option: str, value: Any) -> Optional[str]: def validate_proxy_port(option: str, value: Any) -> Optional[int]: - """Validate a SOCKS5 proxy TCP port.""" + """Validate a SOCKS5 proxy TCP port. + + :raises TypeError: If ``value`` is not an integer or string. + :raises ValueError: If ``value`` is not an integer or is outside the valid port range. + """ if value is None: return None if isinstance(value, bool): @@ -275,6 +283,18 @@ def validate_proxy_port(option: str, value: Any) -> Optional[int]: return port +def validate_proxy_credential(option: str, value: Any) -> Optional[str]: + """Validate a SOCKS5 username or password and its UTF-8 byte length. + + :raises TypeError: If ``value`` is not a string or ``None``. + :raises ValueError: If the UTF-8 byte length exceeds the SOCKS5 limit. + """ + credential = validate_string_or_none(option, value) + if credential is not None and len(credential.encode("utf-8")) > _SOCKS5_MAX_STR_LEN: + raise ValueError(f"{option} must be no more than {_SOCKS5_MAX_STR_LEN} UTF-8 bytes") + return credential + + def validate_readable(option: str, value: Any) -> Optional[str]: """Validates that 'value' is file-like and readable.""" if value is None: @@ -774,8 +794,8 @@ def validate_server_monitoring_mode(option: str, value: str) -> str: "maxstalenessseconds": validate_max_staleness, "proxyhost": validate_proxy_host, "proxyport": validate_proxy_port, - "proxyusername": validate_string_or_none, - "proxypassword": validate_string_or_none, + "proxyusername": validate_proxy_credential, + "proxypassword": validate_proxy_credential, "readconcernlevel": validate_string_or_none, "readpreference": validate_read_preference_mode, "readpreferencetags": validate_read_preference_tags, diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index fdabecbbcb..e0f5eaf1fa 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -228,7 +228,10 @@ def __init__( :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` and a regular function for :class:`~pymongo.synchronous.mongo_client.MongoClient`. Defaults - to ``None``, meaning the driver connects to KMS hosts directly. + to ``None``, meaning the driver uses the encrypted client's SOCKS5 + proxy, or connects directly if none is configured. Cannot be + combined with ``proxyHost`` on the encrypted client; doing so + raises :class:`~pymongo.errors.ConfigurationError`. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 31d214398a..27240b442a 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -42,12 +42,7 @@ from pymongo._telemetry import _CmapTelemetry from pymongo.asynchronous.helpers import _getaddrinfo from pymongo.auth_shared import _AuthContext -from pymongo.common import ( - MAX_BSON_SIZE, - MAX_MESSAGE_SIZE, - MAX_WIRE_VERSION, - MAX_WRITE_BATCH_SIZE, -) +from pymongo.common import MAX_BSON_SIZE, MAX_MESSAGE_SIZE, MAX_WIRE_VERSION, MAX_WRITE_BATCH_SIZE from pymongo.errors import ( # type:ignore[attr-defined] AutoReconnect, ConnectionFailure, @@ -346,7 +341,7 @@ def _handle_connection_error(self, error: BaseException) -> None: if self.is_sdam or type(error) not in (AutoReconnect, NetworkTimeout): return assert isinstance(error, AutoReconnect) # Appease type checker. - # If the original error was a DNS, certificate, or SSL error, ignore it. + # Ignore DNS, certificate, SSL, and SOCKS5 proxy errors. if isinstance( error.__cause__, (_CertificateError, SSLErrors, socket.gaierror, _Socks5Error, _Socks5Timeout), @@ -632,6 +627,7 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s host, port = address if options._proxy_options is not None: + # Import locally to avoid a circular import; socks5_support imports pool_shared. from pymongo.asynchronous.socks5_support import _async_create_proxy_connection return await _async_create_proxy_connection(address, options, options._proxy_options) @@ -641,6 +637,7 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s if not hasattr(socket, "AF_UNIX"): raise ConnectionFailure("UNIX-sockets are not supported on this system") sock = socket.socket(socket.AF_UNIX) + # SOCK_CLOEXEC not supported for Unix sockets. _set_non_inheritable_non_atomic(sock.fileno()) try: sock.setblocking(False) @@ -669,7 +666,6 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s timeout = options.connect_timeout elif timeout <= 0: raise socket.timeout("timed out") - # Socket needs to be non-blocking during connection to not block the event loop await asyncio.wait_for( asyncio.get_running_loop().sock_connect(sock, sa), timeout=timeout ) @@ -870,6 +866,7 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket host, port = address if options._proxy_options is not None: + # Import locally to avoid a circular import; socks5_support imports pool_shared. from pymongo.synchronous.socks5_support import _create_proxy_connection return _create_proxy_connection(address, options, options._proxy_options) diff --git a/pymongo/socks5_shared.py b/pymongo/socks5_shared.py index 6e0ea81d7f..390400752c 100644 --- a/pymongo/socks5_shared.py +++ b/pymongo/socks5_shared.py @@ -40,6 +40,7 @@ _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS = 1 _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS = 2 _SOCKS5_CMD_CONNECT = 0x01 +_SOCKS5_REPLY_SUCCESS = 0x00 _SOCKS5_REPLY_GENERAL_FAILURE = 0x01 _SOCKS5_REPLY_RULESET_DENIED = 0x02 _SOCKS5_REPLY_NETWORK_UNREACHABLE = 0x03 @@ -83,20 +84,18 @@ class _Socks5Timeout(socket.timeout): def _proxy_deadline(options: PoolOptions) -> Optional[float]: - timeout = _csot.remaining() - if timeout is None: - timeout = options.connect_timeout - elif timeout <= 0: - raise _Socks5Timeout("timed out") + timeout = _proxy_socket_timeout(options) if timeout is None: return None return time.monotonic() + timeout def _proxy_socket_timeout(options: PoolOptions) -> Optional[float]: - timeout = _csot.remaining() if _csot.get_timeout() else options.connect_timeout + timeout = _csot.remaining() + if timeout is None: + timeout = options.connect_timeout if timeout is not None and timeout <= 0: - raise _Socks5Timeout("timed out") + raise _Socks5Timeout("SOCKS5 proxy timed out") return timeout @@ -105,7 +104,7 @@ def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: return None remaining = deadline - time.monotonic() if remaining <= 0: - raise _Socks5Timeout("timed out") + raise _Socks5Timeout("SOCKS5 proxy timed out") return remaining @@ -199,7 +198,7 @@ def _socks5_check_connect_reply(data: bytes) -> int: raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") if reserved != _SOCKS5_RESERVED: raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") - if reply: + if reply != _SOCKS5_REPLY_SUCCESS: reason = _SOCKS5_REPLY_REASONS.get(reply, f"unknown error {reply:#04x}") raise _Socks5Error(f"SOCKS5 connect failed: {reason}") if atyp not in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): diff --git a/pymongo/synchronous/_kms_connect.py b/pymongo/synchronous/_kms_connect.py index 78d8119557..7d53a188d9 100644 --- a/pymongo/synchronous/_kms_connect.py +++ b/pymongo/synchronous/_kms_connect.py @@ -63,7 +63,8 @@ def _connect_kms( """Connect to a KMS host and perform the TLS handshake over the socket. Uses ``kms_connect_callback`` when one is provided, otherwise connects - directly, and always verifies against ``address`` (the KMS host). + using the configured SOCKS5 proxy or directly. Always verifies against + ``address`` (the KMS host). """ if kms_connect_callback is None: try: diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 4c8cd8d6c0..05c727fadd 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -52,7 +52,7 @@ from bson.codec_options import CodecOptions from bson.raw_bson import DEFAULT_RAW_BSON_OPTIONS, RawBSONDocument, _inflate_bson from pymongo import _csot, _op_id -from pymongo._kms_connect_shared import KMSConnectCallback +from pymongo._kms_connect_shared import KMSConnectCallback, _check_kms_proxy_options from pymongo.daemon import _spawn_daemon from pymongo.encryption_options import ( AutoEncryptionOpts, @@ -362,6 +362,9 @@ def __init__(self, client: MongoClient[_DocumentTypeArg], opts: AutoEncryptionOp :param client: The encrypted MongoClient. :param opts: The encrypted client's :class:`AutoEncryptionOpts`. """ + _check_kms_proxy_options( + client.options.pool_options._proxy_options, opts._kms_connect_callback + ) if opts._schema_map is None: schema_map = None else: @@ -575,7 +578,8 @@ def __init__( collection. Data keys are protected with encryption by a KMS provider. :param key_vault_client: A MongoClient connected to a MongoDB cluster - containing the `key_vault_namespace` collection. + containing the `key_vault_namespace` collection. Its SOCKS5 proxy + options are also used for KMS requests. :param codec_options: An instance of :class:`~bson.codec_options.CodecOptions` to use when encoding a value for encryption and decoding the decrypted BSON value. This @@ -607,8 +611,11 @@ def __init__( deadline and closes any socket the callback yields later. For an ordinary HTTP proxy, pass :class:`~pymongo.encryption_options.HTTPProxyKMSConnect`. - Defaults to ``None``, meaning the driver connects to KMS hosts - directly. + Defaults to ``None``, meaning the driver uses the SOCKS5 proxy + configured on ``key_vault_client``, or connects directly if none + is configured. Cannot be combined with ``proxyHost`` on + ``key_vault_client``; doing so raises + :class:`~pymongo.errors.ConfigurationError`. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. @@ -638,6 +645,9 @@ def __init__( if not any(cls.__name__ == "MongoClient" for cls in type(key_vault_client).__mro__): raise TypeError(f"MongoClient required but given {type(key_vault_client).__name__}") + _check_kms_proxy_options( + key_vault_client.options.pool_options._proxy_options, kms_connect_callback + ) self._kms_providers = kms_providers self._key_vault_namespace = key_vault_namespace self._key_vault_client = key_vault_client diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 86abd73ddb..fefaec271f 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -337,6 +337,10 @@ def __init__( Defaults to ``20000`` (20 seconds). - `proxyHost` (string): The hostname or IP address of a SOCKS5 proxy. IPv6 literals may be enclosed in brackets. Defaults to no proxy. + Routes MongoDB and automatic encryption KMS connections through + the proxy. Cannot be combined with ``kms_connect_callback`` in + ``auto_encryption_opts``; doing so raises + :class:`~pymongo.errors.ConfigurationError`. - `proxyPort` (integer): The TCP port of the SOCKS5 proxy. Defaults to ``1080`` when ``proxyHost`` is specified. - `proxyUsername` (string): The username for SOCKS5 username/password diff --git a/pymongo/synchronous/socks5_support.py b/pymongo/synchronous/socks5_support.py index 31da3d2375..6853294fef 100644 --- a/pymongo/synchronous/socks5_support.py +++ b/pymongo/synchronous/socks5_support.py @@ -17,7 +17,6 @@ from __future__ import annotations import socket -import time from typing import TYPE_CHECKING, Optional from pymongo.pool_shared import _configure_tcp_socket, _new_tcp_socket @@ -31,6 +30,7 @@ _proxy_connect, _proxy_deadline, _proxy_getaddrinfo, + _proxy_remaining, _proxy_socket_timeout, _socks5_auth_request, _socks5_bound_address_size, @@ -96,7 +96,7 @@ def _create_proxy_connection( try: results = _proxy_getaddrinfo(proxy, family, deadline) except TimeoutError as exc: - raise _Socks5Timeout("timed out") from exc + raise _Socks5Timeout("SOCKS5 proxy timed out") from exc except OSError as exc: raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc @@ -113,14 +113,19 @@ def _create_proxy_connection( return sock except TimeoutError as exc: sock.close() - if connected or (deadline is not None and deadline - time.monotonic() <= 0): - raise _Socks5Timeout("timed out") from exc - err = _Socks5Timeout("timed out") + if connected: + raise _Socks5Timeout("SOCKS5 proxy timed out") from exc + try: + _proxy_remaining(deadline) + except _Socks5Timeout as timeout: + raise timeout from exc + err = _Socks5Timeout("SOCKS5 proxy timed out") err.__cause__ = exc + except _Socks5Error: + sock.close() + raise except OSError as exc: sock.close() - if isinstance(exc, _Socks5Error): - raise err = exc # A connected proxy's handshake failure must not try a different # resolved proxy address; the failure is specific to the endpoint. diff --git a/pymongo/uri_parser_shared.py b/pymongo/uri_parser_shared.py index 745b4c6041..211f04b9d5 100644 --- a/pymongo/uri_parser_shared.py +++ b/pymongo/uri_parser_shared.py @@ -35,13 +35,11 @@ from pymongo.client_options import _parse_ssl_options from pymongo.common import ( - _SOCKS5_MAX_STR_LEN, _URI_OPTS_MUST_RAISE, INTERNAL_URI_OPTION_NAME_MAP, URI_OPTIONS_DEPRECATION_MAP, _CaseInsensitiveDictionary, get_validated_options, - validate_proxy_host, ) from pymongo.errors import ConfigurationError, InvalidURI from pymongo.typings import _Address @@ -493,27 +491,15 @@ def _check_proxy_options(nodes: Iterable[_Address], options: Mapping[str, Any]) proxy_port = options.get("proxyport") proxy_username = options.get("proxyusername") proxy_password = options.get("proxypassword") - if proxy_username == "": - proxy_username = None - if proxy_password == "": - proxy_password = None if proxy_host is None: - if proxy_port is not None or proxy_username is not None or proxy_password is not None: + if proxy_port is not None or proxy_username or proxy_password: raise ConfigurationError( "Cannot specify proxyPort, proxyUsername, or proxyPassword without proxyHost" ) return - # Validate here as well as in the option validator because this function is - # also used by callers that disable URI option validation. - validate_proxy_host("proxyHost", proxy_host) if bool(proxy_username) != bool(proxy_password): raise ConfigurationError("Both proxyUsername and proxyPassword must be specified together") - for name, value in (("proxyUsername", proxy_username), ("proxyPassword", proxy_password)): - if value is not None and len(value.encode("utf-8")) > _SOCKS5_MAX_STR_LEN: - raise ConfigurationError( - f"{name} must be no more than {_SOCKS5_MAX_STR_LEN} UTF-8 bytes" - ) if any(host.endswith(".sock") for host, _ in nodes): raise ConfigurationError("Cannot specify proxyHost with Unix domain sockets") diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index 3991a745af..c3e90f3376 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -25,7 +25,7 @@ from collections.abc import Iterator from contextlib import contextmanager from types import SimpleNamespace -from typing import Any, Optional, cast +from typing import Any, NamedTuple, Optional, cast from unittest import mock from urllib.parse import quote_plus @@ -38,7 +38,21 @@ from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.pool_shared import _async_configured_socket, _async_create_connection -from pymongo.socks5_shared import _Socks5Error +from pymongo.socks5_shared import ( + _SOCKS5_ATYP_DOMAIN, + _SOCKS5_ATYP_IPV4, + _SOCKS5_ATYP_IPV6, + _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, + _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + _SOCKS5_AUTH_NO_ACCEPTABLE, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + _SOCKS5_AUTH_VERSION, + _SOCKS5_CMD_CONNECT, + _SOCKS5_RESERVED, + _SOCKS5_VERSION, + _Socks5Error, +) from test.asynchronous import ( AsyncIntegrationTest, AsyncPyMongoTestCase, @@ -46,21 +60,34 @@ unittest, ) from test.asynchronous.utils import async_wait_until -from test.helpers_shared import CERT_PATH, TLS_OPTIONS +from test.helpers_shared import CERT_PATH, LOCAL_MASTER_KEY, TLS_OPTIONS from test.socks5_helpers import _Socks5Server _IS_SYNC = False pytestmark = pytest.mark.default_async +_SOCKS5_AUTH_GSSAPI_METHOD = 0x01 +_SOCKS5_AUTH_FAILURE_STATUS = 0x01 +_SOCKS4_VERSION = 0x04 + + +class _ProseCase(NamedTuple): + mapped: bool + proxy_port: int + credentials: Optional[tuple[str, str]] + succeeds: bool + @contextmanager -def _refuse_connection(address: tuple[str, int]) -> Iterator[None]: +def _fail_connection( + address: tuple[str, int], error_type: type[OSError] = ConnectionRefusedError +) -> Iterator[None]: if _IS_SYNC: original_connect = socket.socket.connect def connect(sock: socket.socket, target: tuple[str, int]) -> None: if target == address: - raise ConnectionRefusedError("connection refused") + raise error_type("connection failed") original_connect(sock, target) with mock.patch.object(socket.socket, "connect", new=connect): @@ -71,7 +98,7 @@ def connect(sock: socket.socket, target: tuple[str, int]) -> None: async def sock_connect(sock: socket.socket, target: tuple[str, int]) -> None: if target == address: - raise ConnectionRefusedError("connection refused") + raise error_type("connection failed") await original_sock_connect(sock, target) with mock.patch.object(loop, "sock_connect", new=sock_connect): @@ -105,56 +132,105 @@ async def _connect( return sock async def test_greeting_and_authenticated_connect(self) -> None: - server = self._create_server(method=2) + server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) sock = await self._connect(server, ("mongodb.example", 27017), "üser", "påss") sock.close() server.close() - self.assertEqual(server.greeting, b"\x05\x02\x00\x02") - self.assertEqual(server.auth, b"\x01\x05" + "üser".encode() + b"\x05" + "påss".encode()) - self.assertEqual(server.request[:4], b"\x05\x01\x00\x03") + self.assertEqual( + server.greeting, + bytes( + ( + _SOCKS5_VERSION, + _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + ), + ) + self.assertEqual( + server.auth, + bytes((_SOCKS5_AUTH_VERSION, len("üser".encode()))) + + "üser".encode() + + bytes((len("påss".encode()),)) + + "påss".encode(), + ) + self.assertEqual( + server.request[:4], + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED, _SOCKS5_ATYP_DOMAIN)), + ) self.assertEqual( server.request[4:], bytes((len(b"mongodb.example"),)) + b"mongodb.example" + (27017).to_bytes(2, "big"), ) async def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: - server = self._create_server(method=0) + server = self._create_server(method=_SOCKS5_AUTH_NO_AUTH) sock = await self._connect(server, ("127.0.0.1", 27017), "user", "password") sock.close() server.close() - self.assertEqual(server.greeting, b"\x05\x02\x00\x02") + self.assertEqual( + server.greeting, + bytes( + ( + _SOCKS5_VERSION, + _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + ), + ) self.assertEqual(server.auth, b"") - self.assertEqual(server.request, b"\x05\x01\x00\x01\x7f\x00\x00\x01\x69\x89") + self.assertEqual( + server.request, + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED, _SOCKS5_ATYP_IPV4)) + + socket.inet_aton("127.0.0.1") + + (27017).to_bytes(2, "big"), + ) async def test_no_credentials_only_offer_no_authentication(self) -> None: - server = self._create_server(method=2) + server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) with self.assertRaises(_Socks5Error): await self._connect(server, ("localhost", 27017)) server.close() - self.assertEqual(server.greeting, b"\x05\x01\x00") + self.assertEqual( + server.greeting, + bytes( + (_SOCKS5_VERSION, _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, _SOCKS5_AUTH_NO_AUTH) + ), + ) async def test_rejects_all_unacceptable_methods(self) -> None: - for method in (0x01, 0xFF): + for method in (_SOCKS5_AUTH_GSSAPI_METHOD, _SOCKS5_AUTH_NO_ACCEPTABLE): server = self._create_server(method=method) with self.subTest(method=method), self.assertRaises(_Socks5Error): await self._connect(server, ("localhost", 27017)) server.close() async def test_rejects_authentication_failure(self) -> None: - server = self._create_server(method=2, auth_status=1) + server = self._create_server( + method=_SOCKS5_AUTH_USER_PASS, auth_status=_SOCKS5_AUTH_FAILURE_STATUS + ) with self.assertRaises(_Socks5Error): await self._connect(server, ("localhost", 27017), "user", "password") server.close() - self.assertEqual(server.auth, b"\x01\x04user\x08password") + self.assertEqual( + server.auth, + bytes((_SOCKS5_AUTH_VERSION, len(b"user"))) + + b"user" + + bytes((len(b"password"),)) + + b"password", + ) async def test_connect_request_address_types(self) -> None: cases = ( - ("192.0.2.1", b"\x01\xc0\x00\x02\x01"), - ("2001:db8::1", b"\x04" + bytes.fromhex("20010db8000000000000000000000001")), + ("192.0.2.1", bytes((_SOCKS5_ATYP_IPV4,)) + socket.inet_aton("192.0.2.1")), + ( + "2001:db8::1", + bytes((_SOCKS5_ATYP_IPV6,)) + socket.inet_pton(socket.AF_INET6, "2001:db8::1"), + ), ( "münich.example", - b"\x03" - + bytes((len("xn--mnich-kva.example"),)) + bytes((_SOCKS5_ATYP_DOMAIN, len("xn--mnich-kva.example"))) + "xn--mnich-kva.example".encode("ascii"), ), ) @@ -163,10 +239,15 @@ async def test_connect_request_address_types(self) -> None: sock = await self._connect(server, (host, 27017)) sock.close() server.close() - self.assertEqual(server.request, b"\x05\x01\x00" + expected + b"\x69\x89") + self.assertEqual( + server.request, + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) + + expected + + (27017).to_bytes(2, "big"), + ) async def test_drains_bound_address_and_preserves_early_payload(self) -> None: - for atyp in (1, 3, 4): + for atyp in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): server = self._create_server(bound_atyp=atyp, payload=b"mongo") sock = await self._connect(server, ("localhost", 27017)) if _IS_SYNC: @@ -234,11 +315,11 @@ async def test_client_equality_includes_proxy_options(self) -> None: async def test_bad_versions_and_unexpected_eof_fail(self) -> None: cases = ( - (self._create_server(method_version=4), ("localhost", 27017), None), - (self._create_server(reply_version=4), ("localhost", 27017), None), + (self._create_server(method_version=_SOCKS4_VERSION), ("localhost", 27017), None), + (self._create_server(reply_version=_SOCKS4_VERSION), ("localhost", 27017), None), (self._create_server(eof_stage="greeting"), ("localhost", 27017), None), ( - self._create_server(method=2, eof_stage="auth"), + self._create_server(method=_SOCKS5_AUTH_USER_PASS, eof_stage="auth"), ("localhost", 27017), ("user", "password"), ), @@ -252,7 +333,7 @@ async def test_bad_versions_and_unexpected_eof_fail(self) -> None: async def test_timeout_covers_the_entire_handshake(self) -> None: server = self._create_server(stall=True) - with self.assertRaises(socket.timeout): + with self.assertRaisesRegex(socket.timeout, "SOCKS5 proxy timed out"): await self._connect(server, ("localhost", 27017), timeout=0.1) server.close() @@ -284,12 +365,36 @@ async def cancel_after_send(sock: socket.socket, data: bytes) -> None: self.assertEqual(len(sockets), 1) self.assertEqual(sockets[0].fileno(), -1) - @async_client_context.require_async # type: ignore[untyped-decorator] - async def test_tls_sni_targets_destination_not_proxy(self) -> None: + @staticmethod + def _tls_server_context(sni_names: list[Optional[str]]) -> ssl.SSLContext: tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") - sni_names: list[Optional[str]] = [] tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + return tls_context + + async def _assert_tls_sni_targets_destination( + self, target_address: tuple[str, int], sni_names: list[Optional[str]] + ) -> None: + proxy = self._create_server(target=target_address) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + sock = await _async_configured_socket( + ("mongodb.example", target_address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, client_context), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), + ), + ) + self.addAsyncCleanup(sock.close) + sock.close() + self.assertEqual(sni_names, ["mongodb.example"]) + + @unittest.skipIf(_IS_SYNC, "Uses an asynchronous TLS server") + async def test_tls_sni_targets_destination_not_proxy(self) -> None: + sni_names: list[Optional[str]] = [] + tls_context = await asyncio.to_thread(self._tls_server_context, sni_names) async def serve_tls(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: try: @@ -308,28 +413,12 @@ async def close_tls_server() -> None: tls_sockets = tls_server.sockets assert tls_sockets is not None target_address = tls_sockets[0].getsockname() - proxy = self._create_server(target=target_address) - client_context = ssl.create_default_context() - client_context.check_hostname = False - client_context.verify_mode = ssl.CERT_NONE - sock = await _async_configured_socket( - ("mongodb.example", target_address[1]), - PoolOptions( - connect_timeout=2, - ssl_context=cast(Any, client_context), - proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), - ), - ) - self.addAsyncCleanup(sock.close) - sock.close() - self.assertEqual(sni_names, ["mongodb.example"]) + await self._assert_tls_sni_targets_destination(target_address, sni_names) - @async_client_context.require_sync # type: ignore[untyped-decorator] + @unittest.skipUnless(_IS_SYNC, "Uses a synchronous TLS server") async def test_tls_sni_targets_destination_not_proxy_sync(self) -> None: - tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") sni_names: list[Optional[str]] = [] - tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + tls_context = self._tls_server_context(sni_names) listener = socket.socket() listener.bind(("127.0.0.1", 0)) listener.listen(1) @@ -353,21 +442,7 @@ def close_tls_server() -> None: tls_thread.join(timeout=3) self.addAsyncCleanup(close_tls_server) - proxy = self._create_server(target=target_address) - client_context = ssl.create_default_context() - client_context.check_hostname = False - client_context.verify_mode = ssl.CERT_NONE - sock = await _async_configured_socket( - ("mongodb.example", target_address[1]), - PoolOptions( - connect_timeout=2, - ssl_context=cast(Any, client_context), - proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), - ), - ) - self.addAsyncCleanup(sock.close) - sock.close() - self.assertEqual(sni_names, ["mongodb.example"]) + await self._assert_tls_sni_targets_destination(target_address, sni_names) @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") async def test_kms_request_uses_proxy_options(self) -> None: @@ -493,6 +568,11 @@ async def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> Non "mongodb://localhost/?proxyHost=local%20host", "mongodb://localhost/?proxyHost=localhost&proxyPort=0", "mongodb://localhost/?proxyHost=localhost&proxyPort=abc", + "mongodb://localhost/?proxyHost=localhost&proxyUsername=" + + ("é" * 128) + + "&proxyPassword=pass", + "mongodb://localhost/?proxyHost=localhost&proxyUsername=user&proxyPassword=" + + ("é" * 128), ): with self.subTest(uri=uri), self.assertRaises(ValueError): AsyncMongoClient(uri, connect=False) @@ -504,71 +584,161 @@ async def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: from pymongo.socks5_shared import _Socks5Timeout pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) - for cause in (_Socks5Error("boom"), _Socks5Timeout("timed out")): - try: - _raise_connection_failure(("localhost", 27017), cause) - except (AutoReconnect, NetworkTimeout) as wrapped: + self.addAsyncCleanup(pool.close) + for cause, overloaded in ( + (_Socks5Error("boom"), False), + (_Socks5Timeout("SOCKS5 proxy timed out"), False), + (OSError("boom"), True), + ): + with self.subTest(cause=type(cause).__name__): + with self.assertRaises((AutoReconnect, NetworkTimeout)) as raised: + _raise_connection_failure(("localhost", 27017), cause) + wrapped = raised.exception pool._handle_connection_error(wrapped) - self.assertFalse(wrapped.has_error_label("SystemOverloadedError")) - self.assertFalse(wrapped.has_error_label("RetryableError")) + self.assertEqual(wrapped.has_error_label("SystemOverloadedError"), overloaded) + self.assertEqual(wrapped.has_error_label("RetryableError"), overloaded) async def test_multi_address_proxy_fallback_on_connect_error(self) -> None: - server = self._create_server() dead_addr = ("127.0.0.1", 1) - - fake_results = [ - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), - ] target = ( "pymongo.socks5_shared.socket.getaddrinfo" if _IS_SYNC else "pymongo.socks5_shared._getaddrinfo" ) + for error_type in (ConnectionRefusedError, socket.timeout): + server = self._create_server() + fake_results = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), + ] - async def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: - return fake_results + async def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: + return fake_results - replacement: Any = (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo - with mock.patch(target, new=replacement), _refuse_connection(dead_addr): - sock = await self._connect(server, ("localhost", 27017)) - sock.close() + replacement: Any = ( + (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo + ) + with ( + self.subTest(error_type=error_type.__name__), + mock.patch(target, new=replacement), + _fail_connection(dead_addr, error_type), + ): + sock = await self._connect(server, ("localhost", 27017)) + sock.close() + + async def test_proxy_connect_timeout_does_not_retry_after_deadline(self) -> None: + import pymongo.asynchronous.socks5_support as socks5_module + from pymongo.socks5_shared import _Socks5Timeout + + addresses = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)) + for port in (1, 2) + ] + with ( + mock.patch.object(socks5_module, "_proxy_deadline", return_value=0), + mock.patch.object(socks5_module, "_async_proxy_getaddrinfo", return_value=addresses), + mock.patch.object( + socks5_module, "_async_proxy_connect", side_effect=socket.timeout("timed out") + ) as connect, + self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out") as raised, + ): + await _async_create_connection( + ("mongodb.example", 27017), + PoolOptions(proxy_options=_ProxyOptions("proxy.example")), + ) + connect.assert_called_once() + self.assertEqual(connect.call_args.args[0].fileno(), -1) + self.assertIsInstance(raised.exception.__cause__, socket.timeout) @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - async def test_kms_proxy_uses_parent_client_even_with_bypass_auto_encryption(self) -> None: - from pymongo.asynchronous.encryption import _EncryptionIO + async def test_auto_encryption_kms_proxy_uses_encrypted_client(self) -> None: from pymongo.asynchronous.mongo_client import AsyncMongoClient + from pymongo.encryption_options import AutoEncryptionOpts - class FakeOpts: - _kms_connect_callback = None + kv_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + self.addAsyncCleanup(kv_client.close) + for bypass in (False, True): + with self.subTest(bypass_auto_encryption=bypass): + client = AsyncMongoClient( + "mongodb://localhost/?proxyHost=10.0.0.1", + auto_encryption_opts=AutoEncryptionOpts( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + key_vault_client=kv_client, + bypass_auto_encryption=bypass, + ), + connect=False, + ) + self.addAsyncCleanup(client.close) + assert client._encrypter is not None + io_callbacks = client._encrypter._auto_encrypter.callback + self.assertEqual(io_callbacks._proxy_options, _ProxyOptions("10.0.0.1")) + self.assertEqual(io_callbacks.client_ref is None, bypass) + self.assertIsNone( + io_callbacks.mongocryptd_client.options.pool_options._proxy_options + ) - @staticmethod - def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: - return {"aws": object()} + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + async def test_explicit_encryption_kms_proxy_uses_key_vault_client(self) -> None: + from bson.codec_options import CodecOptions + from pymongo.asynchronous.encryption import AsyncClientEncryption + from pymongo.asynchronous.mongo_client import AsyncMongoClient - parent_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.1", connect=False) kv_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - try: - io_bypass = _EncryptionIO( - None, - kv_client.keyvault.datakeys, - None, - FakeOpts(), # type: ignore[arg-type] - proxy_options=parent_client.options.pool_options._proxy_options, + self.addAsyncCleanup(kv_client.close) + encryption = AsyncClientEncryption( + {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", kv_client, CodecOptions() + ) + self.addAsyncCleanup(encryption.close) + assert encryption._io_callbacks is not None + self.assertEqual(encryption._io_callbacks._proxy_options, _ProxyOptions("10.0.0.2")) + + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + async def test_auto_encryption_rejects_kms_callback_with_proxy(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + from pymongo.encryption_options import AutoEncryptionOpts + from pymongo.errors import ConfigurationError + + async def callback(_context: Any) -> socket.socket: + raise AssertionError("KMS callback must not be invoked") + + for bypass in (False, True): + opts = AutoEncryptionOpts( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + bypass_auto_encryption=bypass, + kms_connect_callback=callback, ) - self.assertEqual(io_bypass._proxy_options, _ProxyOptions("10.0.0.1")) - - io_explicit = _EncryptionIO( - None, - kv_client.keyvault.datakeys, - None, - FakeOpts(), # type: ignore[arg-type] - proxy_options=kv_client.options.pool_options._proxy_options, + with ( + self.subTest(bypass_auto_encryption=bypass), + self.assertRaisesRegex(ConfigurationError, "kms_connect_callback with proxyHost"), + ): + AsyncMongoClient( + "mongodb://localhost/?proxyHost=10.0.0.1", + auto_encryption_opts=opts, + connect=False, + ) + + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + async def test_explicit_encryption_rejects_kms_callback_with_proxy(self) -> None: + from bson.codec_options import CodecOptions + from pymongo.asynchronous.encryption import AsyncClientEncryption + from pymongo.asynchronous.mongo_client import AsyncMongoClient + from pymongo.errors import ConfigurationError + + async def callback(_context: Any) -> socket.socket: + raise AssertionError("KMS callback must not be invoked") + + kv_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + self.addAsyncCleanup(kv_client.close) + with self.assertRaisesRegex(ConfigurationError, "kms_connect_callback with proxyHost"): + AsyncClientEncryption( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + kv_client, + CodecOptions(), + kms_connect_callback=callback, ) - self.assertEqual(io_explicit._proxy_options, _ProxyOptions("10.0.0.2")) - finally: - await parent_client.close() - await kv_client.close() async def test_proxy_host_validation(self) -> None: from pymongo.common import validate_proxy_host @@ -587,6 +757,20 @@ async def test_proxy_port_validation(self) -> None: with self.assertRaises(TypeError): validate_proxy_port("proxyPort", True) + async def test_proxy_credential_validation(self) -> None: + from pymongo.common import validate_proxy_credential + + valid = "é" * 127 + "a" + for option in ("proxyUsername", "proxyPassword"): + with self.subTest(option=option): + self.assertIsNone(validate_proxy_credential(option, None)) + self.assertEqual(validate_proxy_credential(option, ""), "") + self.assertEqual(validate_proxy_credential(option, valid), valid) + with self.assertRaisesRegex(ValueError, "no more than 255 UTF-8 bytes"): + validate_proxy_credential(option, "é" * 128) + with self.assertRaises(TypeError): + validate_proxy_credential(option, 1) + async def test_proxy_deadline(self) -> None: from pymongo.socks5_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout @@ -604,12 +788,41 @@ async def test_proxy_socket_timeout_rejects_expired_deadline(self) -> None: from pymongo.socks5_shared import _proxy_socket_timeout, _Socks5Timeout with ( - mock.patch("pymongo.socks5_shared._csot.get_timeout", return_value=True), mock.patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1), self.assertRaises(_Socks5Timeout), ): _proxy_socket_timeout(PoolOptions(connect_timeout=1)) + async def test_proxy_timeouts_share_the_same_budget(self) -> None: + from pymongo.socks5_shared import _proxy_deadline, _proxy_socket_timeout + + options = PoolOptions(connect_timeout=2) + for remaining, expected in ((None, 2), (0.5, 0.5)): + with ( + self.subTest(remaining=remaining), + mock.patch("pymongo.socks5_shared._csot.remaining", return_value=remaining), + mock.patch("pymongo.socks5_shared.time.monotonic", return_value=100), + ): + self.assertEqual(_proxy_socket_timeout(options), expected) + self.assertEqual(_proxy_deadline(options), 100 + expected) + + async def test_proxy_dns_timeout_identifies_the_proxy(self) -> None: + from pymongo.socks5_shared import _Socks5Timeout + + target = ( + "pymongo.socks5_shared.socket.getaddrinfo" + if _IS_SYNC + else "pymongo.socks5_shared._getaddrinfo" + ) + with ( + mock.patch(target, side_effect=socket.timeout("timed out")), + self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), + ): + await _async_create_connection( + ("mongodb.example", 27017), + PoolOptions(connect_timeout=1, proxy_options=_ProxyOptions("proxy.example")), + ) + async def test_socks5_request_rejects_long_domain(self) -> None: from pymongo.socks5_shared import _socks5_request @@ -678,7 +891,7 @@ async def dead_async(*_a: Any, **_kw: Any) -> Any: replacement_dead: Any = (lambda *_a, **_kw: dead_results) if _IS_SYNC else dead_async with ( mock.patch(target, new=replacement_dead), - _refuse_connection(dead_addr), + _fail_connection(dead_addr), self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), ): await _async_create_connection(("localhost", 27017), proxy_opts) @@ -705,7 +918,6 @@ async def asyncSetUp(self) -> None: self.skipTest("SOCKS5 proxies must be started by the Evergreen SOCKS5 tasks.") await super().asyncSetUp() self.nodes = sorted(async_client_context.nodes) - self.credentials = getattr(self, "credentials", {}) self.auth_port = 1080 self.noauth_port = 1081 @@ -765,54 +977,74 @@ async def _run_case( async def test_connection_string_prose_table(self) -> None: cases = ( - (True, self.auth_port, None, False), - (True, self.noauth_port, None, True), - (False, self.auth_port, None, False), - (False, self.noauth_port, None, True), - (True, self.auth_port, ("nonexistentuser", "badauth"), False), - (True, self.noauth_port, ("nonexistentuser", "badauth"), True), - (False, self.noauth_port, ("nonexistentuser", "badauth"), True), - (True, self.auth_port, ("username", "p4ssw0rd"), True), - (True, self.noauth_port, None, True), - (False, self.auth_port, ("username", "p4ssw0rd"), True), - (False, self.noauth_port, None, True), + _ProseCase(mapped=True, proxy_port=self.auth_port, credentials=None, succeeds=False), + _ProseCase(mapped=True, proxy_port=self.noauth_port, credentials=None, succeeds=True), + _ProseCase(mapped=False, proxy_port=self.auth_port, credentials=None, succeeds=False), + _ProseCase(mapped=False, proxy_port=self.noauth_port, credentials=None, succeeds=True), + _ProseCase( + mapped=True, + proxy_port=self.auth_port, + credentials=("nonexistentuser", "badauth"), + succeeds=False, + ), + _ProseCase( + mapped=True, + proxy_port=self.noauth_port, + credentials=("nonexistentuser", "badauth"), + succeeds=True, + ), + _ProseCase( + mapped=False, + proxy_port=self.noauth_port, + credentials=("nonexistentuser", "badauth"), + succeeds=True, + ), + _ProseCase( + mapped=True, + proxy_port=self.auth_port, + credentials=("username", "p4ssw0rd"), + succeeds=True, + ), + _ProseCase( + mapped=False, + proxy_port=self.auth_port, + credentials=("username", "p4ssw0rd"), + succeeds=True, + ), ) - mapped_listener = _CommandListener() - rs_listener = _CommandListener() - for index, (mapped, proxy_port, credentials, succeeds) in enumerate(cases): + for case in cases: for use_uri in (True, False): - listener = None - if not use_uri: - if index == 1: - listener = mapped_listener - elif index == 3: - listener = rs_listener with self.subTest( - case=index, - mapped=mapped, - proxy_port=proxy_port, - credentials=credentials, + mapped=case.mapped, + proxy_port=case.proxy_port, + credentials=case.credentials, use_uri=use_uri, ): await self._run_case( - self._hosts(mapped), - proxy_port, - credentials, - succeeds, - use_uri, - listener, + hosts=self._hosts(case.mapped), + port=case.proxy_port, + credentials=case.credentials, + succeeds=case.succeeds, + use_uri=use_uri, ) + + async def test_command_monitoring_does_not_reference_proxy(self) -> None: proxy_addresses = { ("localhost", self.auth_port), ("localhost", self.noauth_port), ("127.0.0.1", self.auth_port), ("127.0.0.1", self.noauth_port), } - self.assertGreaterEqual(len(mapped_listener.connection_ids), 4) - for connection_id in mapped_listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - self.assertEqual(connection_id, ("localhost", 12345)) - self.assertGreaterEqual(len(rs_listener.connection_ids), 4) - for connection_id in rs_listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - self.assertIn(connection_id, self.nodes) + for mapped in (True, False): + listener = _CommandListener() + with self.subTest(mapped=mapped): + await self._run_case( + self._hosts(mapped), self.noauth_port, None, True, False, listener + ) + self.assertGreaterEqual(len(listener.connection_ids), 4) + for connection_id in listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + if mapped: + self.assertEqual(connection_id, ("localhost", 12345)) + else: + self.assertIn(connection_id, self.nodes) diff --git a/test/socks5_helpers.py b/test/socks5_helpers.py index 34eefe766d..759db05dd1 100644 --- a/test/socks5_helpers.py +++ b/test/socks5_helpers.py @@ -21,17 +21,37 @@ import threading from typing import Optional +from pymongo.socks5_shared import ( + _SOCKS5_ATYP_DOMAIN, + _SOCKS5_ATYP_IPV4, + _SOCKS5_ATYP_IPV6, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_REPLY_LEN, + _SOCKS5_AUTH_SUCCESS, + _SOCKS5_AUTH_USER_PASS, + _SOCKS5_AUTH_VERSION, + _SOCKS5_CONNECT_REPLY_LEN, + _SOCKS5_DOMAIN_LENGTH_LEN, + _SOCKS5_GREETING_REPLY_LEN, + _SOCKS5_IPV4_BOUND_LEN, + _SOCKS5_IPV6_BOUND_LEN, + _SOCKS5_PORT_LEN, + _SOCKS5_REPLY_SUCCESS, + _SOCKS5_RESERVED, + _SOCKS5_VERSION, +) + class _Socks5Server: def __init__( self, - method: int = 0, - auth_status: int = 0, - reply: int = 0, - bound_atyp: int = 1, + method: int = _SOCKS5_AUTH_NO_AUTH, + auth_status: int = _SOCKS5_AUTH_SUCCESS, + reply: int = _SOCKS5_REPLY_SUCCESS, + bound_atyp: int = _SOCKS5_ATYP_IPV4, payload: bytes = b"", - method_version: int = 5, - reply_version: int = 5, + method_version: int = _SOCKS5_VERSION, + reply_version: int = _SOCKS5_VERSION, eof_stage: Optional[str] = None, stall: bool = False, target: Optional[tuple[str, int]] = None, @@ -75,20 +95,20 @@ def _serve(self) -> None: try: with self.listener.accept()[0] as conn: conn.settimeout(3) - version, nmethods = self._read(conn, 2) + version, nmethods = self._read(conn, _SOCKS5_GREETING_REPLY_LEN) self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) self.started.set() if self.stall: conn.recv(1) return if self.eof_stage == "greeting": - conn.sendall(b"\x05") + conn.sendall(bytes((_SOCKS5_VERSION,))) return conn.sendall(bytes((self.method_version, self.method))) - if self.method == 2: - auth_version, username_length = self._read(conn, 2) + if self.method == _SOCKS5_AUTH_USER_PASS: + auth_version, username_length = self._read(conn, _SOCKS5_AUTH_REPLY_LEN) username = self._read(conn, username_length) - password_length = self._read(conn, 1)[0] + password_length = self._read(conn, _SOCKS5_DOMAIN_LENGTH_LEN)[0] password = self._read(conn, password_length) self.auth = ( bytes((auth_version, username_length)) @@ -97,34 +117,47 @@ def _serve(self) -> None: + password ) if self.eof_stage == "auth": - conn.sendall(b"\x01") + conn.sendall(bytes((_SOCKS5_AUTH_VERSION,))) return - conn.sendall(bytes((1, self.auth_status))) + conn.sendall(bytes((_SOCKS5_AUTH_VERSION, self.auth_status))) if self.auth_status: return - version, command, reserved, atyp = self._read(conn, 4) - if atyp == 1: - addr = self._read(conn, 4) - elif atyp == 4: - addr = self._read(conn, 16) + version, command, reserved, atyp = self._read(conn, _SOCKS5_CONNECT_REPLY_LEN) + if atyp == _SOCKS5_ATYP_IPV4: + addr = self._read(conn, _SOCKS5_IPV4_BOUND_LEN - _SOCKS5_PORT_LEN) + elif atyp == _SOCKS5_ATYP_IPV6: + addr = self._read(conn, _SOCKS5_IPV6_BOUND_LEN - _SOCKS5_PORT_LEN) else: - length = self._read(conn, 1) + length = self._read(conn, _SOCKS5_DOMAIN_LENGTH_LEN) addr = length + self._read(conn, length[0]) self.request = ( - bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) + bytes((version, command, reserved, atyp)) + + addr + + self._read(conn, _SOCKS5_PORT_LEN) ) if self.eof_stage == "connect": - conn.sendall(b"\x05\x00") + conn.sendall(bytes((_SOCKS5_VERSION, _SOCKS5_REPLY_SUCCESS))) return if self.target is not None: upstream = socket.create_connection(self.target, timeout=3) - if self.bound_atyp == 1: - bound = b"\x01\x7f\x00\x00\x01\x00\x00" - elif self.bound_atyp == 4: - bound = b"\x04" + (b"\x00" * 16) + b"\x00\x00" + if self.bound_atyp == _SOCKS5_ATYP_IPV4: + bound = ( + bytes((_SOCKS5_ATYP_IPV4,)) + + socket.inet_aton("127.0.0.1") + + bytes(_SOCKS5_PORT_LEN) + ) + elif self.bound_atyp == _SOCKS5_ATYP_IPV6: + bound = ( + bytes((_SOCKS5_ATYP_IPV6,)) + + socket.inet_pton(socket.AF_INET6, "::1") + + bytes(_SOCKS5_PORT_LEN) + ) else: - bound = b"\x03\x01x\x00\x00" - conn.sendall(bytes((self.reply_version, self.reply, 0)) + bound + self.payload) + host = b"x" + bound = bytes((_SOCKS5_ATYP_DOMAIN, len(host))) + host + bytes(_SOCKS5_PORT_LEN) + conn.sendall( + bytes((self.reply_version, self.reply, _SOCKS5_RESERVED)) + bound + self.payload + ) if self.payload: return if upstream is not None: diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index c9b5d84772..362d3a08f8 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -55,7 +55,7 @@ KMSConnectContext, ) from pymongo.errors import ConfigurationError, ConnectionFailure, EncryptionError, NetworkTimeout -from pymongo.pool_options import PoolOptions +from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.ssl_support import get_ssl_context from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM @@ -1058,6 +1058,17 @@ async def test_datagram_socket_from_callback_is_rejected(api): await api.connect(_KMS_ADDRESS, _pool_options(), api.callback_returning(left), 10.0) +@both_apis +async def test_encrypter_rejects_callback_with_socks5_proxy(api): + callback = mock.Mock() + opts = AutoEncryptionOpts({}, "keyvault.datakeys", kms_connect_callback=callback) + client = mock.Mock( + options=mock.Mock(pool_options=PoolOptions(proxy_options=_ProxyOptions("proxy.example"))) + ) + with pytest.raises(ConfigurationError, match="kms_connect_callback with proxyHost"): + api.encryption()._Encrypter(client, opts) + + @both_apis async def test_kms_request_does_not_retry_a_contract_violation(api): # The no-retry guarantee lives in kms_request, not _connect_kms. @@ -1068,9 +1079,7 @@ def action(context): return "not-a-socket" opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=api.callback(action)) - proxy_client = mock.MagicMock() - proxy_client.options.pool_options._proxy_options = None - io = api.encryption()._EncryptionIO(None, mock.MagicMock(), None, opts, proxy_client) + io = api.encryption()._EncryptionIO(None, mock.MagicMock(), None, opts, None) class StubKmsContext: endpoint = "kms.example.com:443" diff --git a/test/test_socks5.py b/test/test_socks5.py index b83025c5d1..02e7b0b9f2 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -25,7 +25,7 @@ from collections.abc import Iterator from contextlib import contextmanager from types import SimpleNamespace -from typing import Any, Optional, cast +from typing import Any, NamedTuple, Optional, cast from unittest import mock from urllib.parse import quote_plus @@ -38,29 +38,56 @@ from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.pool_shared import _configured_socket, _create_connection -from pymongo.socks5_shared import _Socks5Error +from pymongo.socks5_shared import ( + _SOCKS5_ATYP_DOMAIN, + _SOCKS5_ATYP_IPV4, + _SOCKS5_ATYP_IPV6, + _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, + _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + _SOCKS5_AUTH_NO_ACCEPTABLE, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + _SOCKS5_AUTH_VERSION, + _SOCKS5_CMD_CONNECT, + _SOCKS5_RESERVED, + _SOCKS5_VERSION, + _Socks5Error, +) from test import ( IntegrationTest, PyMongoTestCase, client_context, unittest, ) -from test.helpers_shared import CERT_PATH, TLS_OPTIONS +from test.helpers_shared import CERT_PATH, LOCAL_MASTER_KEY, TLS_OPTIONS from test.socks5_helpers import _Socks5Server from test.utils import wait_until _IS_SYNC = True pytestmark = pytest.mark.default +_SOCKS5_AUTH_GSSAPI_METHOD = 0x01 +_SOCKS5_AUTH_FAILURE_STATUS = 0x01 +_SOCKS4_VERSION = 0x04 + + +class _ProseCase(NamedTuple): + mapped: bool + proxy_port: int + credentials: Optional[tuple[str, str]] + succeeds: bool + @contextmanager -def _refuse_connection(address: tuple[str, int]) -> Iterator[None]: +def _fail_connection( + address: tuple[str, int], error_type: type[OSError] = ConnectionRefusedError +) -> Iterator[None]: if _IS_SYNC: original_connect = socket.socket.connect def connect(sock: socket.socket, target: tuple[str, int]) -> None: if target == address: - raise ConnectionRefusedError("connection refused") + raise error_type("connection failed") original_connect(sock, target) with mock.patch.object(socket.socket, "connect", new=connect): @@ -71,7 +98,7 @@ def connect(sock: socket.socket, target: tuple[str, int]) -> None: def sock_connect(sock: socket.socket, target: tuple[str, int]) -> None: if target == address: - raise ConnectionRefusedError("connection refused") + raise error_type("connection failed") original_sock_connect(sock, target) with mock.patch.object(loop, "sock_connect", new=sock_connect): @@ -105,56 +132,105 @@ def _connect( return sock def test_greeting_and_authenticated_connect(self) -> None: - server = self._create_server(method=2) + server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) sock = self._connect(server, ("mongodb.example", 27017), "üser", "påss") sock.close() server.close() - self.assertEqual(server.greeting, b"\x05\x02\x00\x02") - self.assertEqual(server.auth, b"\x01\x05" + "üser".encode() + b"\x05" + "påss".encode()) - self.assertEqual(server.request[:4], b"\x05\x01\x00\x03") + self.assertEqual( + server.greeting, + bytes( + ( + _SOCKS5_VERSION, + _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + ), + ) + self.assertEqual( + server.auth, + bytes((_SOCKS5_AUTH_VERSION, len("üser".encode()))) + + "üser".encode() + + bytes((len("påss".encode()),)) + + "påss".encode(), + ) + self.assertEqual( + server.request[:4], + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED, _SOCKS5_ATYP_DOMAIN)), + ) self.assertEqual( server.request[4:], bytes((len(b"mongodb.example"),)) + b"mongodb.example" + (27017).to_bytes(2, "big"), ) def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: - server = self._create_server(method=0) + server = self._create_server(method=_SOCKS5_AUTH_NO_AUTH) sock = self._connect(server, ("127.0.0.1", 27017), "user", "password") sock.close() server.close() - self.assertEqual(server.greeting, b"\x05\x02\x00\x02") + self.assertEqual( + server.greeting, + bytes( + ( + _SOCKS5_VERSION, + _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + ), + ) self.assertEqual(server.auth, b"") - self.assertEqual(server.request, b"\x05\x01\x00\x01\x7f\x00\x00\x01\x69\x89") + self.assertEqual( + server.request, + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED, _SOCKS5_ATYP_IPV4)) + + socket.inet_aton("127.0.0.1") + + (27017).to_bytes(2, "big"), + ) def test_no_credentials_only_offer_no_authentication(self) -> None: - server = self._create_server(method=2) + server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) with self.assertRaises(_Socks5Error): self._connect(server, ("localhost", 27017)) server.close() - self.assertEqual(server.greeting, b"\x05\x01\x00") + self.assertEqual( + server.greeting, + bytes( + (_SOCKS5_VERSION, _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, _SOCKS5_AUTH_NO_AUTH) + ), + ) def test_rejects_all_unacceptable_methods(self) -> None: - for method in (0x01, 0xFF): + for method in (_SOCKS5_AUTH_GSSAPI_METHOD, _SOCKS5_AUTH_NO_ACCEPTABLE): server = self._create_server(method=method) with self.subTest(method=method), self.assertRaises(_Socks5Error): self._connect(server, ("localhost", 27017)) server.close() def test_rejects_authentication_failure(self) -> None: - server = self._create_server(method=2, auth_status=1) + server = self._create_server( + method=_SOCKS5_AUTH_USER_PASS, auth_status=_SOCKS5_AUTH_FAILURE_STATUS + ) with self.assertRaises(_Socks5Error): self._connect(server, ("localhost", 27017), "user", "password") server.close() - self.assertEqual(server.auth, b"\x01\x04user\x08password") + self.assertEqual( + server.auth, + bytes((_SOCKS5_AUTH_VERSION, len(b"user"))) + + b"user" + + bytes((len(b"password"),)) + + b"password", + ) def test_connect_request_address_types(self) -> None: cases = ( - ("192.0.2.1", b"\x01\xc0\x00\x02\x01"), - ("2001:db8::1", b"\x04" + bytes.fromhex("20010db8000000000000000000000001")), + ("192.0.2.1", bytes((_SOCKS5_ATYP_IPV4,)) + socket.inet_aton("192.0.2.1")), + ( + "2001:db8::1", + bytes((_SOCKS5_ATYP_IPV6,)) + socket.inet_pton(socket.AF_INET6, "2001:db8::1"), + ), ( "münich.example", - b"\x03" - + bytes((len("xn--mnich-kva.example"),)) + bytes((_SOCKS5_ATYP_DOMAIN, len("xn--mnich-kva.example"))) + "xn--mnich-kva.example".encode("ascii"), ), ) @@ -163,10 +239,15 @@ def test_connect_request_address_types(self) -> None: sock = self._connect(server, (host, 27017)) sock.close() server.close() - self.assertEqual(server.request, b"\x05\x01\x00" + expected + b"\x69\x89") + self.assertEqual( + server.request, + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) + + expected + + (27017).to_bytes(2, "big"), + ) def test_drains_bound_address_and_preserves_early_payload(self) -> None: - for atyp in (1, 3, 4): + for atyp in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): server = self._create_server(bound_atyp=atyp, payload=b"mongo") sock = self._connect(server, ("localhost", 27017)) if _IS_SYNC: @@ -234,11 +315,11 @@ def test_client_equality_includes_proxy_options(self) -> None: def test_bad_versions_and_unexpected_eof_fail(self) -> None: cases = ( - (self._create_server(method_version=4), ("localhost", 27017), None), - (self._create_server(reply_version=4), ("localhost", 27017), None), + (self._create_server(method_version=_SOCKS4_VERSION), ("localhost", 27017), None), + (self._create_server(reply_version=_SOCKS4_VERSION), ("localhost", 27017), None), (self._create_server(eof_stage="greeting"), ("localhost", 27017), None), ( - self._create_server(method=2, eof_stage="auth"), + self._create_server(method=_SOCKS5_AUTH_USER_PASS, eof_stage="auth"), ("localhost", 27017), ("user", "password"), ), @@ -252,7 +333,7 @@ def test_bad_versions_and_unexpected_eof_fail(self) -> None: def test_timeout_covers_the_entire_handshake(self) -> None: server = self._create_server(stall=True) - with self.assertRaises(socket.timeout): + with self.assertRaisesRegex(socket.timeout, "SOCKS5 proxy timed out"): self._connect(server, ("localhost", 27017), timeout=0.1) server.close() @@ -284,12 +365,36 @@ def cancel_after_send(sock: socket.socket, data: bytes) -> None: self.assertEqual(len(sockets), 1) self.assertEqual(sockets[0].fileno(), -1) - @client_context.require_async # type: ignore[untyped-decorator] - def test_tls_sni_targets_destination_not_proxy(self) -> None: + @staticmethod + def _tls_server_context(sni_names: list[Optional[str]]) -> ssl.SSLContext: tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") - sni_names: list[Optional[str]] = [] tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + return tls_context + + def _assert_tls_sni_targets_destination( + self, target_address: tuple[str, int], sni_names: list[Optional[str]] + ) -> None: + proxy = self._create_server(target=target_address) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + sock = _configured_socket( + ("mongodb.example", target_address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, client_context), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), + ), + ) + self.addCleanup(sock.close) + sock.close() + self.assertEqual(sni_names, ["mongodb.example"]) + + @unittest.skipIf(_IS_SYNC, "Uses a TLS server") + def test_tls_sni_targets_destination_not_proxy(self) -> None: + sni_names: list[Optional[str]] = [] + tls_context = asyncio.to_thread(self._tls_server_context, sni_names) def serve_tls(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: try: @@ -308,28 +413,12 @@ def close_tls_server() -> None: tls_sockets = tls_server.sockets assert tls_sockets is not None target_address = tls_sockets[0].getsockname() - proxy = self._create_server(target=target_address) - client_context = ssl.create_default_context() - client_context.check_hostname = False - client_context.verify_mode = ssl.CERT_NONE - sock = _configured_socket( - ("mongodb.example", target_address[1]), - PoolOptions( - connect_timeout=2, - ssl_context=cast(Any, client_context), - proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), - ), - ) - self.addCleanup(sock.close) - sock.close() - self.assertEqual(sni_names, ["mongodb.example"]) + self._assert_tls_sni_targets_destination(target_address, sni_names) - @client_context.require_sync # type: ignore[untyped-decorator] + @unittest.skipUnless(_IS_SYNC, "Uses a synchronous TLS server") def test_tls_sni_targets_destination_not_proxy_sync(self) -> None: - tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") sni_names: list[Optional[str]] = [] - tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) + tls_context = self._tls_server_context(sni_names) listener = socket.socket() listener.bind(("127.0.0.1", 0)) listener.listen(1) @@ -353,21 +442,7 @@ def close_tls_server() -> None: tls_thread.join(timeout=3) self.addCleanup(close_tls_server) - proxy = self._create_server(target=target_address) - client_context = ssl.create_default_context() - client_context.check_hostname = False - client_context.verify_mode = ssl.CERT_NONE - sock = _configured_socket( - ("mongodb.example", target_address[1]), - PoolOptions( - connect_timeout=2, - ssl_context=cast(Any, client_context), - proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), - ), - ) - self.addCleanup(sock.close) - sock.close() - self.assertEqual(sni_names, ["mongodb.example"]) + self._assert_tls_sni_targets_destination(target_address, sni_names) @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") def test_kms_request_uses_proxy_options(self) -> None: @@ -493,6 +568,11 @@ def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> None: "mongodb://localhost/?proxyHost=local%20host", "mongodb://localhost/?proxyHost=localhost&proxyPort=0", "mongodb://localhost/?proxyHost=localhost&proxyPort=abc", + "mongodb://localhost/?proxyHost=localhost&proxyUsername=" + + ("é" * 128) + + "&proxyPassword=pass", + "mongodb://localhost/?proxyHost=localhost&proxyUsername=user&proxyPassword=" + + ("é" * 128), ): with self.subTest(uri=uri), self.assertRaises(ValueError): MongoClient(uri, connect=False) @@ -504,71 +584,161 @@ def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: from pymongo.synchronous.pool import Pool pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) - for cause in (_Socks5Error("boom"), _Socks5Timeout("timed out")): - try: - _raise_connection_failure(("localhost", 27017), cause) - except (AutoReconnect, NetworkTimeout) as wrapped: + self.addCleanup(pool.close) + for cause, overloaded in ( + (_Socks5Error("boom"), False), + (_Socks5Timeout("SOCKS5 proxy timed out"), False), + (OSError("boom"), True), + ): + with self.subTest(cause=type(cause).__name__): + with self.assertRaises((AutoReconnect, NetworkTimeout)) as raised: + _raise_connection_failure(("localhost", 27017), cause) + wrapped = raised.exception pool._handle_connection_error(wrapped) - self.assertFalse(wrapped.has_error_label("SystemOverloadedError")) - self.assertFalse(wrapped.has_error_label("RetryableError")) + self.assertEqual(wrapped.has_error_label("SystemOverloadedError"), overloaded) + self.assertEqual(wrapped.has_error_label("RetryableError"), overloaded) def test_multi_address_proxy_fallback_on_connect_error(self) -> None: - server = self._create_server() dead_addr = ("127.0.0.1", 1) - - fake_results = [ - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), - ] target = ( "pymongo.socks5_shared.socket.getaddrinfo" if _IS_SYNC else "pymongo.socks5_shared._getaddrinfo" ) + for error_type in (ConnectionRefusedError, socket.timeout): + server = self._create_server() + fake_results = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), + ] - def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: - return fake_results + def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: + return fake_results - replacement: Any = (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo - with mock.patch(target, new=replacement), _refuse_connection(dead_addr): - sock = self._connect(server, ("localhost", 27017)) - sock.close() + replacement: Any = ( + (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo + ) + with ( + self.subTest(error_type=error_type.__name__), + mock.patch(target, new=replacement), + _fail_connection(dead_addr, error_type), + ): + sock = self._connect(server, ("localhost", 27017)) + sock.close() + + def test_proxy_connect_timeout_does_not_retry_after_deadline(self) -> None: + import pymongo.synchronous.socks5_support as socks5_module + from pymongo.socks5_shared import _Socks5Timeout + + addresses = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)) + for port in (1, 2) + ] + with ( + mock.patch.object(socks5_module, "_proxy_deadline", return_value=0), + mock.patch.object(socks5_module, "_proxy_getaddrinfo", return_value=addresses), + mock.patch.object( + socks5_module, "_proxy_connect", side_effect=socket.timeout("timed out") + ) as connect, + self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out") as raised, + ): + _create_connection( + ("mongodb.example", 27017), + PoolOptions(proxy_options=_ProxyOptions("proxy.example")), + ) + connect.assert_called_once() + self.assertEqual(connect.call_args.args[0].fileno(), -1) + self.assertIsInstance(raised.exception.__cause__, socket.timeout) @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - def test_kms_proxy_uses_parent_client_even_with_bypass_auto_encryption(self) -> None: - from pymongo.synchronous.encryption import _EncryptionIO + def test_auto_encryption_kms_proxy_uses_encrypted_client(self) -> None: + from pymongo.encryption_options import AutoEncryptionOpts from pymongo.synchronous.mongo_client import MongoClient - class FakeOpts: - _kms_connect_callback = None + kv_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + self.addCleanup(kv_client.close) + for bypass in (False, True): + with self.subTest(bypass_auto_encryption=bypass): + client = MongoClient( + "mongodb://localhost/?proxyHost=10.0.0.1", + auto_encryption_opts=AutoEncryptionOpts( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + key_vault_client=kv_client, + bypass_auto_encryption=bypass, + ), + connect=False, + ) + self.addCleanup(client.close) + assert client._encrypter is not None + io_callbacks = client._encrypter._auto_encrypter.callback + self.assertEqual(io_callbacks._proxy_options, _ProxyOptions("10.0.0.1")) + self.assertEqual(io_callbacks.client_ref is None, bypass) + self.assertIsNone( + io_callbacks.mongocryptd_client.options.pool_options._proxy_options + ) - @staticmethod - def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: - return {"aws": object()} + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + def test_explicit_encryption_kms_proxy_uses_key_vault_client(self) -> None: + from bson.codec_options import CodecOptions + from pymongo.synchronous.encryption import ClientEncryption + from pymongo.synchronous.mongo_client import MongoClient - parent_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.1", connect=False) kv_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - try: - io_bypass = _EncryptionIO( - None, - kv_client.keyvault.datakeys, - None, - FakeOpts(), # type: ignore[arg-type] - proxy_options=parent_client.options.pool_options._proxy_options, + self.addCleanup(kv_client.close) + encryption = ClientEncryption( + {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", kv_client, CodecOptions() + ) + self.addCleanup(encryption.close) + assert encryption._io_callbacks is not None + self.assertEqual(encryption._io_callbacks._proxy_options, _ProxyOptions("10.0.0.2")) + + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + def test_auto_encryption_rejects_kms_callback_with_proxy(self) -> None: + from pymongo.encryption_options import AutoEncryptionOpts + from pymongo.errors import ConfigurationError + from pymongo.synchronous.mongo_client import MongoClient + + def callback(_context: Any) -> socket.socket: + raise AssertionError("KMS callback must not be invoked") + + for bypass in (False, True): + opts = AutoEncryptionOpts( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + bypass_auto_encryption=bypass, + kms_connect_callback=callback, ) - self.assertEqual(io_bypass._proxy_options, _ProxyOptions("10.0.0.1")) - - io_explicit = _EncryptionIO( - None, - kv_client.keyvault.datakeys, - None, - FakeOpts(), # type: ignore[arg-type] - proxy_options=kv_client.options.pool_options._proxy_options, + with ( + self.subTest(bypass_auto_encryption=bypass), + self.assertRaisesRegex(ConfigurationError, "kms_connect_callback with proxyHost"), + ): + MongoClient( + "mongodb://localhost/?proxyHost=10.0.0.1", + auto_encryption_opts=opts, + connect=False, + ) + + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + def test_explicit_encryption_rejects_kms_callback_with_proxy(self) -> None: + from bson.codec_options import CodecOptions + from pymongo.errors import ConfigurationError + from pymongo.synchronous.encryption import ClientEncryption + from pymongo.synchronous.mongo_client import MongoClient + + def callback(_context: Any) -> socket.socket: + raise AssertionError("KMS callback must not be invoked") + + kv_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + self.addCleanup(kv_client.close) + with self.assertRaisesRegex(ConfigurationError, "kms_connect_callback with proxyHost"): + ClientEncryption( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + kv_client, + CodecOptions(), + kms_connect_callback=callback, ) - self.assertEqual(io_explicit._proxy_options, _ProxyOptions("10.0.0.2")) - finally: - parent_client.close() - kv_client.close() def test_proxy_host_validation(self) -> None: from pymongo.common import validate_proxy_host @@ -587,6 +757,20 @@ def test_proxy_port_validation(self) -> None: with self.assertRaises(TypeError): validate_proxy_port("proxyPort", True) + def test_proxy_credential_validation(self) -> None: + from pymongo.common import validate_proxy_credential + + valid = "é" * 127 + "a" + for option in ("proxyUsername", "proxyPassword"): + with self.subTest(option=option): + self.assertIsNone(validate_proxy_credential(option, None)) + self.assertEqual(validate_proxy_credential(option, ""), "") + self.assertEqual(validate_proxy_credential(option, valid), valid) + with self.assertRaisesRegex(ValueError, "no more than 255 UTF-8 bytes"): + validate_proxy_credential(option, "é" * 128) + with self.assertRaises(TypeError): + validate_proxy_credential(option, 1) + def test_proxy_deadline(self) -> None: from pymongo.socks5_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout @@ -604,12 +788,41 @@ def test_proxy_socket_timeout_rejects_expired_deadline(self) -> None: from pymongo.socks5_shared import _proxy_socket_timeout, _Socks5Timeout with ( - mock.patch("pymongo.socks5_shared._csot.get_timeout", return_value=True), mock.patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1), self.assertRaises(_Socks5Timeout), ): _proxy_socket_timeout(PoolOptions(connect_timeout=1)) + def test_proxy_timeouts_share_the_same_budget(self) -> None: + from pymongo.socks5_shared import _proxy_deadline, _proxy_socket_timeout + + options = PoolOptions(connect_timeout=2) + for remaining, expected in ((None, 2), (0.5, 0.5)): + with ( + self.subTest(remaining=remaining), + mock.patch("pymongo.socks5_shared._csot.remaining", return_value=remaining), + mock.patch("pymongo.socks5_shared.time.monotonic", return_value=100), + ): + self.assertEqual(_proxy_socket_timeout(options), expected) + self.assertEqual(_proxy_deadline(options), 100 + expected) + + def test_proxy_dns_timeout_identifies_the_proxy(self) -> None: + from pymongo.socks5_shared import _Socks5Timeout + + target = ( + "pymongo.socks5_shared.socket.getaddrinfo" + if _IS_SYNC + else "pymongo.socks5_shared._getaddrinfo" + ) + with ( + mock.patch(target, side_effect=socket.timeout("timed out")), + self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), + ): + _create_connection( + ("mongodb.example", 27017), + PoolOptions(connect_timeout=1, proxy_options=_ProxyOptions("proxy.example")), + ) + def test_socks5_request_rejects_long_domain(self) -> None: from pymongo.socks5_shared import _socks5_request @@ -678,7 +891,7 @@ def dead_async(*_a: Any, **_kw: Any) -> Any: replacement_dead: Any = (lambda *_a, **_kw: dead_results) if _IS_SYNC else dead_async with ( mock.patch(target, new=replacement_dead), - _refuse_connection(dead_addr), + _fail_connection(dead_addr), self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), ): _create_connection(("localhost", 27017), proxy_opts) @@ -705,7 +918,6 @@ def setUp(self) -> None: self.skipTest("SOCKS5 proxies must be started by the Evergreen SOCKS5 tasks.") super().setUp() self.nodes = sorted(client_context.nodes) - self.credentials = getattr(self, "credentials", {}) self.auth_port = 1080 self.noauth_port = 1081 @@ -765,54 +977,72 @@ def _run_case( def test_connection_string_prose_table(self) -> None: cases = ( - (True, self.auth_port, None, False), - (True, self.noauth_port, None, True), - (False, self.auth_port, None, False), - (False, self.noauth_port, None, True), - (True, self.auth_port, ("nonexistentuser", "badauth"), False), - (True, self.noauth_port, ("nonexistentuser", "badauth"), True), - (False, self.noauth_port, ("nonexistentuser", "badauth"), True), - (True, self.auth_port, ("username", "p4ssw0rd"), True), - (True, self.noauth_port, None, True), - (False, self.auth_port, ("username", "p4ssw0rd"), True), - (False, self.noauth_port, None, True), + _ProseCase(mapped=True, proxy_port=self.auth_port, credentials=None, succeeds=False), + _ProseCase(mapped=True, proxy_port=self.noauth_port, credentials=None, succeeds=True), + _ProseCase(mapped=False, proxy_port=self.auth_port, credentials=None, succeeds=False), + _ProseCase(mapped=False, proxy_port=self.noauth_port, credentials=None, succeeds=True), + _ProseCase( + mapped=True, + proxy_port=self.auth_port, + credentials=("nonexistentuser", "badauth"), + succeeds=False, + ), + _ProseCase( + mapped=True, + proxy_port=self.noauth_port, + credentials=("nonexistentuser", "badauth"), + succeeds=True, + ), + _ProseCase( + mapped=False, + proxy_port=self.noauth_port, + credentials=("nonexistentuser", "badauth"), + succeeds=True, + ), + _ProseCase( + mapped=True, + proxy_port=self.auth_port, + credentials=("username", "p4ssw0rd"), + succeeds=True, + ), + _ProseCase( + mapped=False, + proxy_port=self.auth_port, + credentials=("username", "p4ssw0rd"), + succeeds=True, + ), ) - mapped_listener = _CommandListener() - rs_listener = _CommandListener() - for index, (mapped, proxy_port, credentials, succeeds) in enumerate(cases): + for case in cases: for use_uri in (True, False): - listener = None - if not use_uri: - if index == 1: - listener = mapped_listener - elif index == 3: - listener = rs_listener with self.subTest( - case=index, - mapped=mapped, - proxy_port=proxy_port, - credentials=credentials, + mapped=case.mapped, + proxy_port=case.proxy_port, + credentials=case.credentials, use_uri=use_uri, ): self._run_case( - self._hosts(mapped), - proxy_port, - credentials, - succeeds, - use_uri, - listener, + hosts=self._hosts(case.mapped), + port=case.proxy_port, + credentials=case.credentials, + succeeds=case.succeeds, + use_uri=use_uri, ) + + def test_command_monitoring_does_not_reference_proxy(self) -> None: proxy_addresses = { ("localhost", self.auth_port), ("localhost", self.noauth_port), ("127.0.0.1", self.auth_port), ("127.0.0.1", self.noauth_port), } - self.assertGreaterEqual(len(mapped_listener.connection_ids), 4) - for connection_id in mapped_listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - self.assertEqual(connection_id, ("localhost", 12345)) - self.assertGreaterEqual(len(rs_listener.connection_ids), 4) - for connection_id in rs_listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - self.assertIn(connection_id, self.nodes) + for mapped in (True, False): + listener = _CommandListener() + with self.subTest(mapped=mapped): + self._run_case(self._hosts(mapped), self.noauth_port, None, True, False, listener) + self.assertGreaterEqual(len(listener.connection_ids), 4) + for connection_id in listener.connection_ids: + self.assertNotIn(connection_id, proxy_addresses) + if mapped: + self.assertEqual(connection_id, ("localhost", 12345)) + else: + self.assertIn(connection_id, self.nodes) diff --git a/test/test_uri_parser.py b/test/test_uri_parser.py index 1a0a3b22d1..e9ec13a111 100644 --- a/test/test_uri_parser.py +++ b/test/test_uri_parser.py @@ -700,6 +700,37 @@ def test_socks5_invalid_proxy_options(self): with self.subTest(option=option), self.assertRaises(InvalidURI): parse_uri(uri) + def test_socks5_skips_individual_validation_when_disabled(self): + username = "é" * 128 + parsed = parse_uri( + "mongodb://localhost/?proxyHost=%5B%3A%3A1%5D&proxyPort=invalid" + f"&proxyUsername={username}&proxyPassword=pass", + validate=False, + ) + self.assertEqual( + parsed["options"], + { + "proxyHost": "[::1]", + "proxyPort": "invalid", + "proxyUsername": username, + "proxyPassword": "pass", + }, + ) + self.assertEqual( + parse_uri("mongodb://localhost/?proxyHost=invalid%2Fhost", validate=False)["options"], + {"proxyHost": "invalid/host"}, + ) + + def test_socks5_empty_credentials_without_normalization(self): + for validate in (True, False): + with self.subTest(validate=validate): + parsed = parse_uri( + "mongodb://localhost/?proxyUsername=&proxyPassword=", + validate=validate, + normalize=False, + ) + self.assertEqual(parsed["options"], {"proxyUsername": "", "proxyPassword": ""}) + def test_socks5_proxy_rejects_unix_socket(self): with self.assertRaises(ConfigurationError): parse_uri("mongodb://%2Ftmp%2Fmongodb.sock/?proxyHost=localhost") From f9a3b4fa1d6bf29f49fb9ada676901203b5025d1 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Sat, 7 Nov 2026 10:12:08 +0000 Subject: [PATCH 14/16] address new review --- test/asynchronous/test_socks5.py | 401 ++++--------------------------- test/socks5_helpers.py | 32 +++ test/test_kms_connect.py | 122 +++++++++- test/test_socks5.py | 401 ++++--------------------------- test/test_uri_parser.py | 78 ++++++ 5 files changed, 330 insertions(+), 704 deletions(-) diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py index c3e90f3376..9ab3288477 100644 --- a/test/asynchronous/test_socks5.py +++ b/test/asynchronous/test_socks5.py @@ -21,10 +21,8 @@ import socket import ssl import sys -import threading from collections.abc import Iterator from contextlib import contextmanager -from types import SimpleNamespace from typing import Any, NamedTuple, Optional, cast from unittest import mock from urllib.parse import quote_plus @@ -34,7 +32,6 @@ sys.path[0:0] = [""] from pymongo import monitoring -from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.pool_shared import _async_configured_socket, _async_create_connection @@ -52,6 +49,7 @@ _SOCKS5_RESERVED, _SOCKS5_VERSION, _Socks5Error, + _Socks5Timeout, ) from test.asynchronous import ( AsyncIntegrationTest, @@ -60,8 +58,8 @@ unittest, ) from test.asynchronous.utils import async_wait_until -from test.helpers_shared import CERT_PATH, LOCAL_MASTER_KEY, TLS_OPTIONS -from test.socks5_helpers import _Socks5Server +from test.helpers_shared import TLS_OPTIONS +from test.socks5_helpers import _Socks5Server, _TLSServer _IS_SYNC = False pytestmark = pytest.mark.default_async @@ -365,22 +363,16 @@ async def cancel_after_send(sock: socket.socket, data: bytes) -> None: self.assertEqual(len(sockets), 1) self.assertEqual(sockets[0].fileno(), -1) - @staticmethod - def _tls_server_context(sni_names: list[Optional[str]]) -> ssl.SSLContext: - tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") - tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) - return tls_context - - async def _assert_tls_sni_targets_destination( - self, target_address: tuple[str, int], sni_names: list[Optional[str]] - ) -> None: - proxy = self._create_server(target=target_address) + async def test_tls_sni_targets_destination_not_proxy(self) -> None: + sni_names: list[Optional[str]] = [] + tls_server = _TLSServer(sni_names) + self.addAsyncCleanup(tls_server.close) + proxy = self._create_server(target=tls_server.address) client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) client_context.check_hostname = False client_context.verify_mode = ssl.CERT_NONE sock = await _async_configured_socket( - ("mongodb.example", target_address[1]), + ("mongodb.example", tls_server.address[1]), PoolOptions( connect_timeout=2, ssl_context=cast(Any, client_context), @@ -391,118 +383,6 @@ async def _assert_tls_sni_targets_destination( sock.close() self.assertEqual(sni_names, ["mongodb.example"]) - @unittest.skipIf(_IS_SYNC, "Uses an asynchronous TLS server") - async def test_tls_sni_targets_destination_not_proxy(self) -> None: - sni_names: list[Optional[str]] = [] - tls_context = await asyncio.to_thread(self._tls_server_context, sni_names) - - async def serve_tls(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - await reader.read(1) - finally: - writer.close() - await writer.wait_closed() - - tls_server: Any = await asyncio.start_server(serve_tls, "127.0.0.1", 0, ssl=tls_context) - - async def close_tls_server() -> None: - tls_server.close() - await tls_server.wait_closed() - - self.addAsyncCleanup(close_tls_server) - tls_sockets = tls_server.sockets - assert tls_sockets is not None - target_address = tls_sockets[0].getsockname() - await self._assert_tls_sni_targets_destination(target_address, sni_names) - - @unittest.skipUnless(_IS_SYNC, "Uses a synchronous TLS server") - async def test_tls_sni_targets_destination_not_proxy_sync(self) -> None: - sni_names: list[Optional[str]] = [] - tls_context = self._tls_server_context(sni_names) - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - target_address = listener.getsockname() - - def serve_tls() -> None: - try: - raw, _ = listener.accept() - with tls_context.wrap_socket(raw, server_side=True) as conn: - conn.recv(1) - except OSError: - pass - finally: - listener.close() - - tls_thread = threading.Thread(target=serve_tls, daemon=True) - tls_thread.start() - - def close_tls_server() -> None: - listener.close() - tls_thread.join(timeout=3) - - self.addAsyncCleanup(close_tls_server) - await self._assert_tls_sni_targets_destination(target_address, sni_names) - - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - async def test_kms_request_uses_proxy_options(self) -> None: - import pymongo.asynchronous.encryption as encryption_module - from pymongo.asynchronous.encryption import _EncryptionIO - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - class FakeOpts: - _kms_connect_callback = None - - @staticmethod - def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: - return {"aws": object()} - - client = AsyncMongoClient("mongodb://localhost/", connect=False) - try: - for expected in (None, _ProxyOptions("127.0.0.1")): - with self.subTest(proxy_options=expected): - io = _EncryptionIO( - client, - client.keyvault.datakeys, - None, - FakeOpts(), # type: ignore[arg-type] - proxy_options=expected, - ) - connection = mock.MagicMock() - - async def connect(_address, options, _callback, _timeout): - self.assertIs(options._proxy_options, expected) - return connection - - async def sendall(_connection, _message): - pass - - with ( - mock.patch.object(encryption_module, "_connect_kms", new=connect), - mock.patch.object(encryption_module, "async_socket_sendall", new=sendall), - ): - await io.kms_request( - SimpleNamespace( - endpoint="kms.example:443", - message=b"request", - kms_provider="aws", - usleep=0, - bytes_needed=0, - ) - ) - connection.close.assert_called_once_with() - finally: - await client.close() - - async def test_proxy_settings_are_stored_in_pool_options(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - client = AsyncMongoClient("mongodb://localhost/?proxyHost=[::1]", connect=False) - try: - self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1")) - finally: - await client.close() - async def test_proxy_password_is_redacted_in_repr(self) -> None: from pymongo.asynchronous.mongo_client import AsyncMongoClient @@ -559,24 +439,6 @@ async def test_split_proxy_options_across_uri_and_kwargs(self) -> None: await client1.close() await client2.close() - async def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - for uri in ( - "mongodb://localhost/?proxyHost=", - "mongodb://localhost/?proxyHost=proxy%3A1080", - "mongodb://localhost/?proxyHost=local%20host", - "mongodb://localhost/?proxyHost=localhost&proxyPort=0", - "mongodb://localhost/?proxyHost=localhost&proxyPort=abc", - "mongodb://localhost/?proxyHost=localhost&proxyUsername=" - + ("é" * 128) - + "&proxyPassword=pass", - "mongodb://localhost/?proxyHost=localhost&proxyUsername=user&proxyPassword=" - + ("é" * 128), - ): - with self.subTest(uri=uri), self.assertRaises(ValueError): - AsyncMongoClient(uri, connect=False) - async def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: from pymongo.asynchronous.pool import Pool from pymongo.errors import AutoReconnect, NetworkTimeout @@ -650,209 +512,6 @@ async def test_proxy_connect_timeout_does_not_retry_after_deadline(self) -> None self.assertEqual(connect.call_args.args[0].fileno(), -1) self.assertIsInstance(raised.exception.__cause__, socket.timeout) - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - async def test_auto_encryption_kms_proxy_uses_encrypted_client(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - from pymongo.encryption_options import AutoEncryptionOpts - - kv_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - self.addAsyncCleanup(kv_client.close) - for bypass in (False, True): - with self.subTest(bypass_auto_encryption=bypass): - client = AsyncMongoClient( - "mongodb://localhost/?proxyHost=10.0.0.1", - auto_encryption_opts=AutoEncryptionOpts( - {"local": {"key": LOCAL_MASTER_KEY}}, - "keyvault.datakeys", - key_vault_client=kv_client, - bypass_auto_encryption=bypass, - ), - connect=False, - ) - self.addAsyncCleanup(client.close) - assert client._encrypter is not None - io_callbacks = client._encrypter._auto_encrypter.callback - self.assertEqual(io_callbacks._proxy_options, _ProxyOptions("10.0.0.1")) - self.assertEqual(io_callbacks.client_ref is None, bypass) - self.assertIsNone( - io_callbacks.mongocryptd_client.options.pool_options._proxy_options - ) - - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - async def test_explicit_encryption_kms_proxy_uses_key_vault_client(self) -> None: - from bson.codec_options import CodecOptions - from pymongo.asynchronous.encryption import AsyncClientEncryption - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - kv_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - self.addAsyncCleanup(kv_client.close) - encryption = AsyncClientEncryption( - {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", kv_client, CodecOptions() - ) - self.addAsyncCleanup(encryption.close) - assert encryption._io_callbacks is not None - self.assertEqual(encryption._io_callbacks._proxy_options, _ProxyOptions("10.0.0.2")) - - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - async def test_auto_encryption_rejects_kms_callback_with_proxy(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - from pymongo.encryption_options import AutoEncryptionOpts - from pymongo.errors import ConfigurationError - - async def callback(_context: Any) -> socket.socket: - raise AssertionError("KMS callback must not be invoked") - - for bypass in (False, True): - opts = AutoEncryptionOpts( - {"local": {"key": LOCAL_MASTER_KEY}}, - "keyvault.datakeys", - bypass_auto_encryption=bypass, - kms_connect_callback=callback, - ) - with ( - self.subTest(bypass_auto_encryption=bypass), - self.assertRaisesRegex(ConfigurationError, "kms_connect_callback with proxyHost"), - ): - AsyncMongoClient( - "mongodb://localhost/?proxyHost=10.0.0.1", - auto_encryption_opts=opts, - connect=False, - ) - - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - async def test_explicit_encryption_rejects_kms_callback_with_proxy(self) -> None: - from bson.codec_options import CodecOptions - from pymongo.asynchronous.encryption import AsyncClientEncryption - from pymongo.asynchronous.mongo_client import AsyncMongoClient - from pymongo.errors import ConfigurationError - - async def callback(_context: Any) -> socket.socket: - raise AssertionError("KMS callback must not be invoked") - - kv_client = AsyncMongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - self.addAsyncCleanup(kv_client.close) - with self.assertRaisesRegex(ConfigurationError, "kms_connect_callback with proxyHost"): - AsyncClientEncryption( - {"local": {"key": LOCAL_MASTER_KEY}}, - "keyvault.datakeys", - kv_client, - CodecOptions(), - kms_connect_callback=callback, - ) - - async def test_proxy_host_validation(self) -> None: - from pymongo.common import validate_proxy_host - - self.assertIsNone(validate_proxy_host("proxyHost", None)) - self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") - with self.assertRaises(ValueError): - validate_proxy_host("proxyHost", "[not-ipv6]") - with self.assertRaises(ValueError): - validate_proxy_host("proxyHost", "bad/host") - - async def test_proxy_port_validation(self) -> None: - from pymongo.common import validate_proxy_port - - self.assertIsNone(validate_proxy_port("proxyPort", None)) - with self.assertRaises(TypeError): - validate_proxy_port("proxyPort", True) - - async def test_proxy_credential_validation(self) -> None: - from pymongo.common import validate_proxy_credential - - valid = "é" * 127 + "a" - for option in ("proxyUsername", "proxyPassword"): - with self.subTest(option=option): - self.assertIsNone(validate_proxy_credential(option, None)) - self.assertEqual(validate_proxy_credential(option, ""), "") - self.assertEqual(validate_proxy_credential(option, valid), valid) - with self.assertRaisesRegex(ValueError, "no more than 255 UTF-8 bytes"): - validate_proxy_credential(option, "é" * 128) - with self.assertRaises(TypeError): - validate_proxy_credential(option, 1) - - async def test_proxy_deadline(self) -> None: - from pymongo.socks5_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout - - self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) - self.assertIsNone(_proxy_remaining(None)) - with self.assertRaises(_Socks5Timeout): - _proxy_remaining(0.0) - with ( - mock.patch("pymongo.socks5_shared._csot.remaining", return_value=0.0), - self.assertRaises(_Socks5Timeout), - ): - _proxy_deadline(PoolOptions(connect_timeout=1)) - - async def test_proxy_socket_timeout_rejects_expired_deadline(self) -> None: - from pymongo.socks5_shared import _proxy_socket_timeout, _Socks5Timeout - - with ( - mock.patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1), - self.assertRaises(_Socks5Timeout), - ): - _proxy_socket_timeout(PoolOptions(connect_timeout=1)) - - async def test_proxy_timeouts_share_the_same_budget(self) -> None: - from pymongo.socks5_shared import _proxy_deadline, _proxy_socket_timeout - - options = PoolOptions(connect_timeout=2) - for remaining, expected in ((None, 2), (0.5, 0.5)): - with ( - self.subTest(remaining=remaining), - mock.patch("pymongo.socks5_shared._csot.remaining", return_value=remaining), - mock.patch("pymongo.socks5_shared.time.monotonic", return_value=100), - ): - self.assertEqual(_proxy_socket_timeout(options), expected) - self.assertEqual(_proxy_deadline(options), 100 + expected) - - async def test_proxy_dns_timeout_identifies_the_proxy(self) -> None: - from pymongo.socks5_shared import _Socks5Timeout - - target = ( - "pymongo.socks5_shared.socket.getaddrinfo" - if _IS_SYNC - else "pymongo.socks5_shared._getaddrinfo" - ) - with ( - mock.patch(target, side_effect=socket.timeout("timed out")), - self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), - ): - await _async_create_connection( - ("mongodb.example", 27017), - PoolOptions(connect_timeout=1, proxy_options=_ProxyOptions("proxy.example")), - ) - - async def test_socks5_request_rejects_long_domain(self) -> None: - from pymongo.socks5_shared import _socks5_request - - with self.assertRaisesRegex(_Socks5Error, "invalid domain name length"): - _socks5_request(("a" * 256, 27017)) - - async def test_socks5_request_rejects_invalid_domain(self) -> None: - from pymongo.socks5_shared import _socks5_request - - with self.assertRaisesRegex(_Socks5Error, "invalid domain name"): - _socks5_request((".münich.example", 27017)) - - async def test_socks5_request_checks_port_before_encoding_domain(self) -> None: - from pymongo.socks5_shared import _socks5_request - - with self.assertRaisesRegex(_Socks5Error, "destination port is required"): - _socks5_request(("a" * 256, None)) # type: ignore[arg-type] - - async def test_socks5_connect_reply_rejects_invalid_reserved_byte(self) -> None: - from pymongo.socks5_shared import _socks5_check_connect_reply - - with self.assertRaisesRegex(_Socks5Error, "invalid reserved byte"): - _socks5_check_connect_reply(b"\x05\x00\x01\x01") - - async def test_socks5_connect_reply_rejects_invalid_address_type(self) -> None: - from pymongo.socks5_shared import _socks5_check_connect_reply - - with self.assertRaisesRegex(_Socks5Error, "invalid address type"): - _socks5_check_connect_reply(b"\x05\x00\x00\x09") - async def test_proxy_connection_error_paths(self) -> None: proxy_opts = PoolOptions( connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080) @@ -872,6 +531,12 @@ def raise_gai(*_a: Any, **_kw: Any) -> Any: ): await _async_create_connection(("localhost", 27017), proxy_opts) + with ( + mock.patch(target, side_effect=socket.timeout("timed out")), + self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), + ): + await _async_create_connection(("localhost", 27017), proxy_opts) + async def empty_async(*_a: Any, **_kw: Any) -> Any: return [] @@ -896,6 +561,42 @@ async def dead_async(*_a: Any, **_kw: Any) -> Any: ): await _async_create_connection(("localhost", 27017), proxy_opts) + with ( + mock.patch(target, new=replacement_dead), + _fail_connection(dead_addr, socket.timeout), + self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), + ): + await _async_create_connection(("localhost", 27017), proxy_opts) + + server = self._create_server() + handshake_options = PoolOptions( + connect_timeout=1, + proxy_options=_ProxyOptions(server.address[0], server.address[1]), + ) + sendall_target = ( + "pymongo.synchronous.socks5_support._socks5_sendall" + if _IS_SYNC + else "pymongo.asynchronous.socks5_support._async_socks5_sendall" + ) + with ( + mock.patch(sendall_target, side_effect=OSError("handshake failed")), + self.assertRaisesRegex(_Socks5Error, "SOCKS5 handshake failed"), + ): + await _async_create_connection(("localhost", 27017), handshake_options) + server.close() + + server = self._create_server() + handshake_options = PoolOptions( + connect_timeout=1, + proxy_options=_ProxyOptions(server.address[0], server.address[1]), + ) + with ( + mock.patch(sendall_target, side_effect=KeyboardInterrupt), + self.assertRaises(KeyboardInterrupt), + ): + await _async_create_connection(("localhost", 27017), handshake_options) + server.close() + class _CommandListener(monitoring.CommandListener): def __init__(self) -> None: diff --git a/test/socks5_helpers.py b/test/socks5_helpers.py index 759db05dd1..577afa7bb9 100644 --- a/test/socks5_helpers.py +++ b/test/socks5_helpers.py @@ -18,6 +18,7 @@ import select import socket +import ssl import threading from typing import Optional @@ -40,6 +41,37 @@ _SOCKS5_RESERVED, _SOCKS5_VERSION, ) +from test.helpers_shared import CERT_PATH + + +class _TLSServer: + def __init__(self, sni_names: list[Optional[str]]) -> None: + self.sni_names = sni_names + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.listener.settimeout(3) + self.address = self.listener.getsockname() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + def _serve(self) -> None: + try: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(f"{CERT_PATH}/server.pem") + context.set_servername_callback(lambda _sock, name, _ctx: self.sni_names.append(name)) + with self.listener.accept()[0] as raw: + with context.wrap_socket(raw, server_side=True) as conn: + conn.recv(1) + except OSError: + pass + finally: + self.listener.close() + + def close(self) -> None: + self.listener.close() + self.thread.join(timeout=3) class _Socks5Server: diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index 362d3a08f8..29439ab636 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -40,6 +40,7 @@ from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from types import SimpleNamespace from typing import Any from unittest import mock @@ -57,7 +58,7 @@ from pymongo.errors import ConfigurationError, ConnectionFailure, EncryptionError, NetworkTimeout from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.ssl_support import get_ssl_context -from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM +from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM, LOCAL_MASTER_KEY pytestmark = [pytest.mark.encryption, pytest.mark.asyncio] @@ -167,15 +168,15 @@ def client_encryption(self, kms_providers, key_vault_namespace, client, kms_conn kms_connect_callback=kms_connect_callback, ) - def simple_client(self): + def simple_client(self, *args, **kwargs): """A lazily-connecting client for this API.""" if self.is_async: from pymongo.asynchronous.mongo_client import AsyncMongoClient - return AsyncMongoClient() + return AsyncMongoClient(*args, **kwargs) from pymongo import MongoClient - return MongoClient() + return MongoClient(*args, **kwargs) ASYNC = Facade(is_async=True) @@ -1058,6 +1059,106 @@ async def test_datagram_socket_from_callback_is_rejected(api): await api.connect(_KMS_ADDRESS, _pool_options(), api.callback_returning(left), 10.0) +@both_apis +async def test_kms_request_uses_proxy_options(api): + if not _HAVE_PYMONGOCRYPT: + pytest.skip("pymongocrypt is not installed") + encryption_module = api.encryption() + + class FakeOpts: + _kms_connect_callback = None + + @staticmethod + def _kms_ssl_contexts(_is_sync): + return {"aws": object()} + + client = api.simple_client("mongodb://localhost/", connect=False) + try: + for expected in (None, _ProxyOptions("127.0.0.1")): + with pytest.MonkeyPatch.context() as monkeypatch: + io = encryption_module._EncryptionIO( + client, + client.keyvault.datakeys, + None, + FakeOpts(), + proxy_options=expected, + ) + connection = mock.MagicMock() + + async def connect(_address, options, _callback, _timeout): + assert options._proxy_options is expected + return connection + + async def sendall(_connection, _message): + pass + + if api.is_async: + monkeypatch.setattr(encryption_module, "_connect_kms", connect) + monkeypatch.setattr(encryption_module, "async_socket_sendall", sendall) + else: + + def sync_connect(_address, options, _callback, _timeout): + assert options._proxy_options is expected + return connection + + monkeypatch.setattr(encryption_module, "_connect_kms", sync_connect) + request = SimpleNamespace( + endpoint="kms.example:443", + message=b"request", + kms_provider="aws", + usleep=0, + bytes_needed=0, + ) + await api.maybe_await(io.kms_request(request)) + connection.close.assert_called_once_with() + finally: + await api.maybe_await(client.close()) + + +@both_apis +@pytest.mark.skipif(not _HAVE_PYMONGOCRYPT, reason="pymongocrypt is not installed") +async def test_auto_encryption_kms_proxy_uses_encrypted_client(api): + kv_client = api.simple_client("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + clients = [] + try: + for bypass in (False, True): + client = api.simple_client( + "mongodb://localhost/?proxyHost=10.0.0.1", + auto_encryption_opts=AutoEncryptionOpts( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + key_vault_client=kv_client, + bypass_auto_encryption=bypass, + ), + connect=False, + ) + clients.append(client) + assert client._encrypter is not None + io_callbacks = client._encrypter._auto_encrypter.callback + assert io_callbacks._proxy_options == _ProxyOptions("10.0.0.1") + assert (io_callbacks.client_ref is None) is bypass + assert io_callbacks.mongocryptd_client.options.pool_options._proxy_options is None + finally: + for client in clients: + await api.maybe_await(client.close()) + await api.maybe_await(kv_client.close()) + + +@both_apis +@pytest.mark.skipif(not _HAVE_PYMONGOCRYPT, reason="pymongocrypt is not installed") +async def test_explicit_encryption_kms_proxy_uses_key_vault_client(api): + kv_client = api.simple_client("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + encryption = api.client_encryption( + {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", kv_client, None + ) + try: + assert encryption._io_callbacks is not None + assert encryption._io_callbacks._proxy_options == _ProxyOptions("10.0.0.2") + finally: + await api.maybe_await(encryption.close()) + await api.maybe_await(kv_client.close()) + + @both_apis async def test_encrypter_rejects_callback_with_socks5_proxy(api): callback = mock.Mock() @@ -1068,6 +1169,19 @@ async def test_encrypter_rejects_callback_with_socks5_proxy(api): with pytest.raises(ConfigurationError, match="kms_connect_callback with proxyHost"): api.encryption()._Encrypter(client, opts) + if _HAVE_PYMONGOCRYPT: + kv_client = api.simple_client("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + try: + with pytest.raises(ConfigurationError, match="kms_connect_callback with proxyHost"): + api.client_encryption( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + kv_client, + api.callback(callback), + ) + finally: + await api.maybe_await(kv_client.close()) + @both_apis async def test_kms_request_does_not_retry_a_contract_violation(api): diff --git a/test/test_socks5.py b/test/test_socks5.py index 02e7b0b9f2..b6f4fc1215 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -21,10 +21,8 @@ import socket import ssl import sys -import threading from collections.abc import Iterator from contextlib import contextmanager -from types import SimpleNamespace from typing import Any, NamedTuple, Optional, cast from unittest import mock from urllib.parse import quote_plus @@ -34,7 +32,6 @@ sys.path[0:0] = [""] from pymongo import monitoring -from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.pool_shared import _configured_socket, _create_connection @@ -52,6 +49,7 @@ _SOCKS5_RESERVED, _SOCKS5_VERSION, _Socks5Error, + _Socks5Timeout, ) from test import ( IntegrationTest, @@ -59,8 +57,8 @@ client_context, unittest, ) -from test.helpers_shared import CERT_PATH, LOCAL_MASTER_KEY, TLS_OPTIONS -from test.socks5_helpers import _Socks5Server +from test.helpers_shared import TLS_OPTIONS +from test.socks5_helpers import _Socks5Server, _TLSServer from test.utils import wait_until _IS_SYNC = True @@ -365,22 +363,16 @@ def cancel_after_send(sock: socket.socket, data: bytes) -> None: self.assertEqual(len(sockets), 1) self.assertEqual(sockets[0].fileno(), -1) - @staticmethod - def _tls_server_context(sni_names: list[Optional[str]]) -> ssl.SSLContext: - tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - tls_context.load_cert_chain(f"{CERT_PATH}/server.pem") - tls_context.set_servername_callback(lambda _sock, name, _ctx: sni_names.append(name)) - return tls_context - - def _assert_tls_sni_targets_destination( - self, target_address: tuple[str, int], sni_names: list[Optional[str]] - ) -> None: - proxy = self._create_server(target=target_address) + def test_tls_sni_targets_destination_not_proxy(self) -> None: + sni_names: list[Optional[str]] = [] + tls_server = _TLSServer(sni_names) + self.addCleanup(tls_server.close) + proxy = self._create_server(target=tls_server.address) client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) client_context.check_hostname = False client_context.verify_mode = ssl.CERT_NONE sock = _configured_socket( - ("mongodb.example", target_address[1]), + ("mongodb.example", tls_server.address[1]), PoolOptions( connect_timeout=2, ssl_context=cast(Any, client_context), @@ -391,118 +383,6 @@ def _assert_tls_sni_targets_destination( sock.close() self.assertEqual(sni_names, ["mongodb.example"]) - @unittest.skipIf(_IS_SYNC, "Uses a TLS server") - def test_tls_sni_targets_destination_not_proxy(self) -> None: - sni_names: list[Optional[str]] = [] - tls_context = asyncio.to_thread(self._tls_server_context, sni_names) - - def serve_tls(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - reader.read(1) - finally: - writer.close() - writer.wait_closed() - - tls_server: Any = asyncio.start_server(serve_tls, "127.0.0.1", 0, ssl=tls_context) - - def close_tls_server() -> None: - tls_server.close() - tls_server.wait_closed() - - self.addCleanup(close_tls_server) - tls_sockets = tls_server.sockets - assert tls_sockets is not None - target_address = tls_sockets[0].getsockname() - self._assert_tls_sni_targets_destination(target_address, sni_names) - - @unittest.skipUnless(_IS_SYNC, "Uses a synchronous TLS server") - def test_tls_sni_targets_destination_not_proxy_sync(self) -> None: - sni_names: list[Optional[str]] = [] - tls_context = self._tls_server_context(sni_names) - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - target_address = listener.getsockname() - - def serve_tls() -> None: - try: - raw, _ = listener.accept() - with tls_context.wrap_socket(raw, server_side=True) as conn: - conn.recv(1) - except OSError: - pass - finally: - listener.close() - - tls_thread = threading.Thread(target=serve_tls, daemon=True) - tls_thread.start() - - def close_tls_server() -> None: - listener.close() - tls_thread.join(timeout=3) - - self.addCleanup(close_tls_server) - self._assert_tls_sni_targets_destination(target_address, sni_names) - - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - def test_kms_request_uses_proxy_options(self) -> None: - import pymongo.synchronous.encryption as encryption_module - from pymongo.synchronous.encryption import _EncryptionIO - from pymongo.synchronous.mongo_client import MongoClient - - class FakeOpts: - _kms_connect_callback = None - - @staticmethod - def _kms_ssl_contexts(_is_sync: bool) -> dict[str, object]: - return {"aws": object()} - - client = MongoClient("mongodb://localhost/", connect=False) - try: - for expected in (None, _ProxyOptions("127.0.0.1")): - with self.subTest(proxy_options=expected): - io = _EncryptionIO( - client, - client.keyvault.datakeys, - None, - FakeOpts(), # type: ignore[arg-type] - proxy_options=expected, - ) - connection = mock.MagicMock() - - def connect(_address, options, _callback, _timeout): - self.assertIs(options._proxy_options, expected) - return connection - - def sendall(_connection, _message): - pass - - with ( - mock.patch.object(encryption_module, "_connect_kms", new=connect), - mock.patch.object(encryption_module, "sendall", new=sendall), - ): - io.kms_request( - SimpleNamespace( - endpoint="kms.example:443", - message=b"request", - kms_provider="aws", - usleep=0, - bytes_needed=0, - ) - ) - connection.close.assert_called_once_with() - finally: - client.close() - - def test_proxy_settings_are_stored_in_pool_options(self) -> None: - from pymongo.synchronous.mongo_client import MongoClient - - client = MongoClient("mongodb://localhost/?proxyHost=[::1]", connect=False) - try: - self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1")) - finally: - client.close() - def test_proxy_password_is_redacted_in_repr(self) -> None: from pymongo.synchronous.mongo_client import MongoClient @@ -559,24 +439,6 @@ def test_split_proxy_options_across_uri_and_kwargs(self) -> None: client1.close() client2.close() - def test_malformed_proxy_uri_options_raise_instead_of_warning(self) -> None: - from pymongo.synchronous.mongo_client import MongoClient - - for uri in ( - "mongodb://localhost/?proxyHost=", - "mongodb://localhost/?proxyHost=proxy%3A1080", - "mongodb://localhost/?proxyHost=local%20host", - "mongodb://localhost/?proxyHost=localhost&proxyPort=0", - "mongodb://localhost/?proxyHost=localhost&proxyPort=abc", - "mongodb://localhost/?proxyHost=localhost&proxyUsername=" - + ("é" * 128) - + "&proxyPassword=pass", - "mongodb://localhost/?proxyHost=localhost&proxyUsername=user&proxyPassword=" - + ("é" * 128), - ): - with self.subTest(uri=uri), self.assertRaises(ValueError): - MongoClient(uri, connect=False) - def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: from pymongo.errors import AutoReconnect, NetworkTimeout from pymongo.pool_shared import _raise_connection_failure @@ -650,209 +512,6 @@ def test_proxy_connect_timeout_does_not_retry_after_deadline(self) -> None: self.assertEqual(connect.call_args.args[0].fileno(), -1) self.assertIsInstance(raised.exception.__cause__, socket.timeout) - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - def test_auto_encryption_kms_proxy_uses_encrypted_client(self) -> None: - from pymongo.encryption_options import AutoEncryptionOpts - from pymongo.synchronous.mongo_client import MongoClient - - kv_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - self.addCleanup(kv_client.close) - for bypass in (False, True): - with self.subTest(bypass_auto_encryption=bypass): - client = MongoClient( - "mongodb://localhost/?proxyHost=10.0.0.1", - auto_encryption_opts=AutoEncryptionOpts( - {"local": {"key": LOCAL_MASTER_KEY}}, - "keyvault.datakeys", - key_vault_client=kv_client, - bypass_auto_encryption=bypass, - ), - connect=False, - ) - self.addCleanup(client.close) - assert client._encrypter is not None - io_callbacks = client._encrypter._auto_encrypter.callback - self.assertEqual(io_callbacks._proxy_options, _ProxyOptions("10.0.0.1")) - self.assertEqual(io_callbacks.client_ref is None, bypass) - self.assertIsNone( - io_callbacks.mongocryptd_client.options.pool_options._proxy_options - ) - - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - def test_explicit_encryption_kms_proxy_uses_key_vault_client(self) -> None: - from bson.codec_options import CodecOptions - from pymongo.synchronous.encryption import ClientEncryption - from pymongo.synchronous.mongo_client import MongoClient - - kv_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - self.addCleanup(kv_client.close) - encryption = ClientEncryption( - {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", kv_client, CodecOptions() - ) - self.addCleanup(encryption.close) - assert encryption._io_callbacks is not None - self.assertEqual(encryption._io_callbacks._proxy_options, _ProxyOptions("10.0.0.2")) - - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - def test_auto_encryption_rejects_kms_callback_with_proxy(self) -> None: - from pymongo.encryption_options import AutoEncryptionOpts - from pymongo.errors import ConfigurationError - from pymongo.synchronous.mongo_client import MongoClient - - def callback(_context: Any) -> socket.socket: - raise AssertionError("KMS callback must not be invoked") - - for bypass in (False, True): - opts = AutoEncryptionOpts( - {"local": {"key": LOCAL_MASTER_KEY}}, - "keyvault.datakeys", - bypass_auto_encryption=bypass, - kms_connect_callback=callback, - ) - with ( - self.subTest(bypass_auto_encryption=bypass), - self.assertRaisesRegex(ConfigurationError, "kms_connect_callback with proxyHost"), - ): - MongoClient( - "mongodb://localhost/?proxyHost=10.0.0.1", - auto_encryption_opts=opts, - connect=False, - ) - - @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") - def test_explicit_encryption_rejects_kms_callback_with_proxy(self) -> None: - from bson.codec_options import CodecOptions - from pymongo.errors import ConfigurationError - from pymongo.synchronous.encryption import ClientEncryption - from pymongo.synchronous.mongo_client import MongoClient - - def callback(_context: Any) -> socket.socket: - raise AssertionError("KMS callback must not be invoked") - - kv_client = MongoClient("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - self.addCleanup(kv_client.close) - with self.assertRaisesRegex(ConfigurationError, "kms_connect_callback with proxyHost"): - ClientEncryption( - {"local": {"key": LOCAL_MASTER_KEY}}, - "keyvault.datakeys", - kv_client, - CodecOptions(), - kms_connect_callback=callback, - ) - - def test_proxy_host_validation(self) -> None: - from pymongo.common import validate_proxy_host - - self.assertIsNone(validate_proxy_host("proxyHost", None)) - self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") - with self.assertRaises(ValueError): - validate_proxy_host("proxyHost", "[not-ipv6]") - with self.assertRaises(ValueError): - validate_proxy_host("proxyHost", "bad/host") - - def test_proxy_port_validation(self) -> None: - from pymongo.common import validate_proxy_port - - self.assertIsNone(validate_proxy_port("proxyPort", None)) - with self.assertRaises(TypeError): - validate_proxy_port("proxyPort", True) - - def test_proxy_credential_validation(self) -> None: - from pymongo.common import validate_proxy_credential - - valid = "é" * 127 + "a" - for option in ("proxyUsername", "proxyPassword"): - with self.subTest(option=option): - self.assertIsNone(validate_proxy_credential(option, None)) - self.assertEqual(validate_proxy_credential(option, ""), "") - self.assertEqual(validate_proxy_credential(option, valid), valid) - with self.assertRaisesRegex(ValueError, "no more than 255 UTF-8 bytes"): - validate_proxy_credential(option, "é" * 128) - with self.assertRaises(TypeError): - validate_proxy_credential(option, 1) - - def test_proxy_deadline(self) -> None: - from pymongo.socks5_shared import _proxy_deadline, _proxy_remaining, _Socks5Timeout - - self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) - self.assertIsNone(_proxy_remaining(None)) - with self.assertRaises(_Socks5Timeout): - _proxy_remaining(0.0) - with ( - mock.patch("pymongo.socks5_shared._csot.remaining", return_value=0.0), - self.assertRaises(_Socks5Timeout), - ): - _proxy_deadline(PoolOptions(connect_timeout=1)) - - def test_proxy_socket_timeout_rejects_expired_deadline(self) -> None: - from pymongo.socks5_shared import _proxy_socket_timeout, _Socks5Timeout - - with ( - mock.patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1), - self.assertRaises(_Socks5Timeout), - ): - _proxy_socket_timeout(PoolOptions(connect_timeout=1)) - - def test_proxy_timeouts_share_the_same_budget(self) -> None: - from pymongo.socks5_shared import _proxy_deadline, _proxy_socket_timeout - - options = PoolOptions(connect_timeout=2) - for remaining, expected in ((None, 2), (0.5, 0.5)): - with ( - self.subTest(remaining=remaining), - mock.patch("pymongo.socks5_shared._csot.remaining", return_value=remaining), - mock.patch("pymongo.socks5_shared.time.monotonic", return_value=100), - ): - self.assertEqual(_proxy_socket_timeout(options), expected) - self.assertEqual(_proxy_deadline(options), 100 + expected) - - def test_proxy_dns_timeout_identifies_the_proxy(self) -> None: - from pymongo.socks5_shared import _Socks5Timeout - - target = ( - "pymongo.socks5_shared.socket.getaddrinfo" - if _IS_SYNC - else "pymongo.socks5_shared._getaddrinfo" - ) - with ( - mock.patch(target, side_effect=socket.timeout("timed out")), - self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), - ): - _create_connection( - ("mongodb.example", 27017), - PoolOptions(connect_timeout=1, proxy_options=_ProxyOptions("proxy.example")), - ) - - def test_socks5_request_rejects_long_domain(self) -> None: - from pymongo.socks5_shared import _socks5_request - - with self.assertRaisesRegex(_Socks5Error, "invalid domain name length"): - _socks5_request(("a" * 256, 27017)) - - def test_socks5_request_rejects_invalid_domain(self) -> None: - from pymongo.socks5_shared import _socks5_request - - with self.assertRaisesRegex(_Socks5Error, "invalid domain name"): - _socks5_request((".münich.example", 27017)) - - def test_socks5_request_checks_port_before_encoding_domain(self) -> None: - from pymongo.socks5_shared import _socks5_request - - with self.assertRaisesRegex(_Socks5Error, "destination port is required"): - _socks5_request(("a" * 256, None)) # type: ignore[arg-type] - - def test_socks5_connect_reply_rejects_invalid_reserved_byte(self) -> None: - from pymongo.socks5_shared import _socks5_check_connect_reply - - with self.assertRaisesRegex(_Socks5Error, "invalid reserved byte"): - _socks5_check_connect_reply(b"\x05\x00\x01\x01") - - def test_socks5_connect_reply_rejects_invalid_address_type(self) -> None: - from pymongo.socks5_shared import _socks5_check_connect_reply - - with self.assertRaisesRegex(_Socks5Error, "invalid address type"): - _socks5_check_connect_reply(b"\x05\x00\x00\x09") - def test_proxy_connection_error_paths(self) -> None: proxy_opts = PoolOptions( connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080) @@ -872,6 +531,12 @@ def raise_gai(*_a: Any, **_kw: Any) -> Any: ): _create_connection(("localhost", 27017), proxy_opts) + with ( + mock.patch(target, side_effect=socket.timeout("timed out")), + self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), + ): + _create_connection(("localhost", 27017), proxy_opts) + def empty_async(*_a: Any, **_kw: Any) -> Any: return [] @@ -896,6 +561,42 @@ def dead_async(*_a: Any, **_kw: Any) -> Any: ): _create_connection(("localhost", 27017), proxy_opts) + with ( + mock.patch(target, new=replacement_dead), + _fail_connection(dead_addr, socket.timeout), + self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), + ): + _create_connection(("localhost", 27017), proxy_opts) + + server = self._create_server() + handshake_options = PoolOptions( + connect_timeout=1, + proxy_options=_ProxyOptions(server.address[0], server.address[1]), + ) + sendall_target = ( + "pymongo.synchronous.socks5_support._socks5_sendall" + if _IS_SYNC + else "pymongo.socks5_support._socks5_sendall" + ) + with ( + mock.patch(sendall_target, side_effect=OSError("handshake failed")), + self.assertRaisesRegex(_Socks5Error, "SOCKS5 handshake failed"), + ): + _create_connection(("localhost", 27017), handshake_options) + server.close() + + server = self._create_server() + handshake_options = PoolOptions( + connect_timeout=1, + proxy_options=_ProxyOptions(server.address[0], server.address[1]), + ) + with ( + mock.patch(sendall_target, side_effect=KeyboardInterrupt), + self.assertRaises(KeyboardInterrupt), + ): + _create_connection(("localhost", 27017), handshake_options) + server.close() + class _CommandListener(monitoring.CommandListener): def __init__(self) -> None: diff --git a/test/test_uri_parser.py b/test/test_uri_parser.py index e9ec13a111..c0c4897792 100644 --- a/test/test_uri_parser.py +++ b/test/test_uri_parser.py @@ -30,7 +30,19 @@ from bson.binary import JAVA_LEGACY from pymongo import ReadPreference +from pymongo.common import validate_proxy_credential, validate_proxy_host, validate_proxy_port from pymongo.errors import ConfigurationError, InvalidURI +from pymongo.pool_options import PoolOptions +from pymongo.socks5_shared import ( + _proxy_deadline, + _proxy_remaining, + _proxy_socket_timeout, + _socks5_bound_address_size, + _socks5_check_connect_reply, + _socks5_request, + _Socks5Error, + _Socks5Timeout, +) from pymongo.synchronous.uri_parser import parse_uri from pymongo.uri_parser_shared import ( _unquoted_percent, @@ -731,6 +743,72 @@ def test_socks5_empty_credentials_without_normalization(self): ) self.assertEqual(parsed["options"], {"proxyUsername": "", "proxyPassword": ""}) + def test_socks5_validator_functions(self): + self.assertIsNone(validate_proxy_host("proxyHost", None)) + self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") + with self.assertRaises(ValueError): + validate_proxy_host("proxyHost", "[not-ipv6]") + with self.assertRaises(ValueError): + validate_proxy_host("proxyHost", "bad/host") + + self.assertIsNone(validate_proxy_port("proxyPort", None)) + self.assertEqual(validate_proxy_port("proxyPort", 1080), 1080) + with self.assertRaises(TypeError): + validate_proxy_port("proxyPort", True) + + valid = "é" * 127 + "a" + for option in ("proxyUsername", "proxyPassword"): + with self.subTest(option=option): + self.assertIsNone(validate_proxy_credential(option, None)) + self.assertEqual(validate_proxy_credential(option, ""), "") + self.assertEqual(validate_proxy_credential(option, valid), valid) + with self.assertRaisesRegex(ValueError, "no more than 255 UTF-8 bytes"): + validate_proxy_credential(option, "é" * 128) + with self.assertRaises(TypeError): + validate_proxy_credential(option, 1) + + def test_socks5_deadline_helpers(self): + self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) + self.assertIsNone(_proxy_remaining(None)) + with self.assertRaises(_Socks5Timeout): + _proxy_remaining(0.0) + with ( + patch("pymongo.socks5_shared._csot.remaining", return_value=0.0), + self.assertRaises(_Socks5Timeout), + ): + _proxy_deadline(PoolOptions(connect_timeout=1)) + with ( + patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1), + self.assertRaises(_Socks5Timeout), + ): + _proxy_socket_timeout(PoolOptions(connect_timeout=1)) + for remaining, expected in ((None, 2), (0.5, 0.5)): + with ( + self.subTest(remaining=remaining), + patch("pymongo.socks5_shared._csot.remaining", return_value=remaining), + patch("pymongo.socks5_shared.time.monotonic", return_value=100), + ): + options = PoolOptions(connect_timeout=2) + self.assertEqual(_proxy_socket_timeout(options), expected) + self.assertEqual(_proxy_deadline(options), 100 + expected) + + def test_socks5_protocol_validation(self): + with self.assertRaisesRegex(_Socks5Error, "missing bound domain length"): + _socks5_bound_address_size(3) + for address, message in ( + (("a" * 256, 27017), "invalid domain name length"), + ((".münich.example", 27017), "invalid domain name"), + (("a" * 256, None), "destination port is required"), + ): + with self.subTest(address=address), self.assertRaisesRegex(_Socks5Error, message): + _socks5_request(address) # type: ignore[arg-type] + for reply, message in ( + (b"\x05\x00\x01\x01", "invalid reserved byte"), + (b"\x05\x00\x00\x09", "invalid address type"), + ): + with self.subTest(reply=reply), self.assertRaisesRegex(_Socks5Error, message): + _socks5_check_connect_reply(reply) + def test_socks5_proxy_rejects_unix_socket(self): with self.assertRaises(ConfigurationError): parse_uri("mongodb://%2Ftmp%2Fmongodb.sock/?proxyHost=localhost") From 10bc033596b71863386d99b6d990af5b2d9492ac Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Sat, 7 Nov 2026 10:59:47 +0000 Subject: [PATCH 15/16] reduce patch size significantly --- .evergreen/scripts/generate-config.sh | 2 +- .evergreen/scripts/run_tests.py | 4 +- .evergreen/scripts/setup_tests.py | 92 -- .evergreen/scripts/teardown_tests.py | 29 - .evergreen/scripts/test_generate_config.py | 1 - .evergreen/scripts/test_socks5_proxy.py | 175 --- pymongo/asynchronous/socks5_support.py | 18 +- pymongo/socks5_shared.py | 32 +- pymongo/synchronous/socks5_support.py | 20 +- test/asynchronous/test_socks5.py | 751 ------------ test/socks5_helpers.py | 220 ---- test/test_kms_connect.py | 170 +-- test/test_socks5.py | 1205 ++++++++++++-------- test/test_uri_parser.py | 159 +-- 14 files changed, 821 insertions(+), 2057 deletions(-) delete mode 100644 .evergreen/scripts/test_socks5_proxy.py delete mode 100644 test/asynchronous/test_socks5.py delete mode 100644 test/socks5_helpers.py diff --git a/.evergreen/scripts/generate-config.sh b/.evergreen/scripts/generate-config.sh index 669a3d3d59..4f46605901 100755 --- a/.evergreen/scripts/generate-config.sh +++ b/.evergreen/scripts/generate-config.sh @@ -4,6 +4,6 @@ set -eu # Assert the generation rules, then regenerate the config. -python -m pytest .evergreen/scripts/test_generate_config.py .evergreen/scripts/test_socks5_proxy.py -m "" +python -m pytest .evergreen/scripts/test_generate_config.py -m "" python .evergreen/scripts/generate_config.py diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 632faaf3f8..2af8090a49 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -186,9 +186,7 @@ def handle_fuzz() -> None: def run() -> None: if TEST_NAME == "socks5": - TEST_ARGS.extend( - ["test/test_socks5.py", "test/asynchronous/test_socks5.py", "-k", "TestSocks5Prose"] - ) + TEST_ARGS.extend(["test/test_socks5.py", "-k", "TestSocks5Prose"]) # Add diagnostic for python version. print("Running with python", sys.version) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index ddfdd45098..9e687c9cab 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -4,15 +4,11 @@ import os import platform import shutil -import socket import stat -import subprocess import sys -import time import zipfile from pathlib import Path from urllib import request -from urllib.parse import urlsplit from utils import ( DRIVERS_TOOLS, @@ -56,13 +52,6 @@ "pyopenssl": "ocsp", } -SOCKS5_AUTH_PORT = 1080 -SOCKS5_NOAUTH_PORT = 1081 -SOCKS5_MAPPED_PORT = 12345 -SOCKS5_PROXY_STARTUP_TIMEOUT = 10 -SOCKS5_PROXY_POLL_INTERVAL = 0.1 - - # Map the test name to test group. GROUP_MAP = dict(fuzz="fuzz", mockupdb="mockupdb", perf="perf") @@ -187,84 +176,6 @@ def get_secrets(name: str) -> dict[str, str]: return load_config_from_file(secrets_dir / "secrets-export.sh") -def _first_seed(uri: str) -> str: - authority = urlsplit(uri).netloc.rsplit("@", 1)[-1] - seed = urlsplit(f"mongodb://{authority.split(',', 1)[0]}") - host = seed.hostname - if not host: - raise ValueError(f"Invalid seed in MONGODB_URI: {authority}") - port = seed.port if seed.port is not None else 27017 - return f"[{host}]:{port}" if ":" in host else f"{host}:{port}" - - -def _wait_for_socks5_proxy(process: subprocess.Popen[bytes], port: int, log_path: Path) -> None: - """Wait for a SOCKS5 proxy to accept connections. - - :param process: The proxy process to monitor. - :param port: The local port on which the proxy should listen. - :param log_path: The proxy's log file, included in errors. - :raises RuntimeError: If the process exits or does not become ready in time. - """ - deadline = time.monotonic() + SOCKS5_PROXY_STARTUP_TIMEOUT - while True: - if process.poll() is not None: - raise RuntimeError(f"SOCKS5 proxy on port {port} exited; see {log_path}") - try: - with socket.create_connection(("127.0.0.1", port), timeout=SOCKS5_PROXY_POLL_INTERVAL): - return - except OSError as exc: - if time.monotonic() >= deadline: - raise RuntimeError( - f"SOCKS5 proxy on port {port} did not start; see {log_path}" - ) from exc - time.sleep(SOCKS5_PROXY_POLL_INTERVAL) - - -def setup_socks5() -> None: - """Start and verify the authenticated and unauthenticated test proxies. - - :raises ValueError: If ``MONGODB_URI`` is unset or invalid. - :raises RuntimeError: If the tools checkout is missing or a proxy fails to start. - """ - check_drivers_tools() - if not os.environ.get("MONGODB_URI"): - raise ValueError("MONGODB_URI must point to the replica set for SOCKS5 tests") - - script = (Path(DRIVERS_TOOLS) / ".evergreen" / "socks5srv.py").absolute() - mapping = f"localhost:{SOCKS5_MAPPED_PORT} to {_first_seed(os.environ['MONGODB_URI'])}" - processes: list[subprocess.Popen[bytes]] = [] - started = False - try: - for port in (SOCKS5_AUTH_PORT, SOCKS5_NOAUTH_PORT): - args = [sys.executable, str(script), "--port", str(port), "--map", mapping] - if port == SOCKS5_AUTH_PORT: - args.extend(("--auth", "username:p4ssw0rd")) - log_path = HERE / f"socks5-proxy-{port}.log" - LOGGER.info("Starting SOCKS5 proxy on port %s; log: %s", port, log_path) - with log_path.open("wb") as log: - process = subprocess.Popen( # noqa: S603 - args, - stdout=log, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - processes.append(process) - _wait_for_socks5_proxy(process, port, log_path) - write_env("SOCKS5_PROXY_PIDS", " ".join(str(process.pid) for process in processes)) - write_env("SOCKS5_PROXIES_STARTED") - started = True - finally: - if not started: - for process in processes: - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - process.kill() - process.wait() - - def handle_test_env() -> None: opts, _ = get_test_options("Set up the test environment and services.") test_name = opts.test_name @@ -423,9 +334,6 @@ def handle_test_env() -> None: cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start' run_command(cmd) - if test_name == "socks5": - setup_socks5() - if test_name == "ocsp": if sub_test_name: os.environ["OCSP_SERVER_TYPE"] = sub_test_name diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index d564dcca2d..09f2925ef5 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -1,10 +1,7 @@ from __future__ import annotations import os -import shlex import shutil -import signal -import subprocess import sys from pathlib import Path @@ -39,32 +36,6 @@ elif TEST_NAME == "ocsp": run_command(f"bash {DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh") -elif TEST_NAME == "socks5": - ps = shutil.which("ps") - if ps is None: - raise RuntimeError("Cannot locate ps to verify SOCKS5 proxy process ownership") - script = str((Path(DRIVERS_TOOLS) / ".evergreen" / "socks5srv.py").absolute()) - for value in os.environ.get("SOCKS5_PROXY_PIDS", "").split(): - pid = int(value) - if pid <= 0: - raise ValueError(f"Invalid SOCKS5 proxy PID: {pid}") - result = subprocess.run( # noqa: S603 - [ps, "-ww", "-p", str(pid), "-o", "args="], - capture_output=True, - text=True, - check=False, - ) - if result.returncode == 1: - LOGGER.info("SOCKS5 proxy PID %s already exited", pid) - continue - result.check_returncode() - if script not in shlex.split(result.stdout): - raise RuntimeError(f"Refusing to stop unrelated process with PID {pid}") - try: - os.kill(pid, signal.SIGTERM) - except ProcessLookupError: - LOGGER.info("SOCKS5 proxy PID %s already exited", pid) - # Tear down atlas cluster if applicable. if TEST_NAME in ["aws_lambda", "search_index"]: run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh") diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index 1d9d2d4701..2fb836369c 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -98,7 +98,6 @@ def test_socks5_variant_selects_both_prose_tasks(): assert variant.display_name == "SOCKS5 Prose Tests" assert "pr" in variant.tags assert variant.expansions["UV_PYTHON"] == generate_config_utils.MIN_PYTHON - assert "SOCKS5_PROXIES_STARTED" not in variant.expansions assert all(variant_selects(variant, task) for task in generate_config.create_socks5_tasks()) diff --git a/.evergreen/scripts/test_socks5_proxy.py b/.evergreen/scripts/test_socks5_proxy.py deleted file mode 100644 index 9a1e36b039..0000000000 --- a/.evergreen/scripts/test_socks5_proxy.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Tests for SOCKS5 test setup, execution, and teardown.""" - -from __future__ import annotations - -import os -import runpy -import signal -import subprocess -import sys -from pathlib import Path -from unittest import mock - -import pytest - -sys.path.insert(0, str(Path(__file__).parent)) - -import run_tests -import setup_tests -from utils import TEST_SUITE_MAP - - -@pytest.fixture -def proxy_setup(monkeypatch, tmp_path): - monkeypatch.setenv("MONGODB_URI", "mongodb://localhost:27017/") - monkeypatch.setattr(setup_tests, "HERE", tmp_path) - first = mock.Mock(pid=11) - second = mock.Mock(pid=22) - for process in (first, second): - process.poll.return_value = None - with ( - mock.patch.object(setup_tests, "check_drivers_tools"), - mock.patch.object(setup_tests, "write_env") as write_env, - mock.patch.object(setup_tests.socket, "create_connection") as connect, - mock.patch.object(setup_tests.subprocess, "Popen", side_effect=[first, second]) as popen, - ): - yield first, second, popen, connect, write_env - - -def test_setup_starts_and_checks_both_proxies(proxy_setup): - first, second, popen, connect, write_env = proxy_setup - setup_tests.setup_socks5() - script = str((Path(setup_tests.DRIVERS_TOOLS) / ".evergreen" / "socks5srv.py").absolute()) - args = [call.args[0] for call in popen.call_args_list] - assert args == [ - [ - sys.executable, - script, - "--port", - "1080", - "--map", - "localhost:12345 to localhost:27017", - "--auth", - "username:p4ssw0rd", - ], - [ - sys.executable, - script, - "--port", - "1081", - "--map", - "localhost:12345 to localhost:27017", - ], - ] - assert connect.call_args_list == [ - mock.call(("127.0.0.1", 1080), timeout=0.1), - mock.call(("127.0.0.1", 1081), timeout=0.1), - ] - assert write_env.call_args_list == [ - mock.call("SOCKS5_PROXY_PIDS", "11 22"), - mock.call("SOCKS5_PROXIES_STARTED"), - ] - first.terminate.assert_not_called() - second.terminate.assert_not_called() - - -def test_setup_cleans_up_after_second_proxy_fails(proxy_setup, monkeypatch): - first, second, _popen, connect, write_env = proxy_setup - connect.side_effect = [mock.MagicMock(), ConnectionRefusedError("not ready")] - monkeypatch.setattr(setup_tests, "time", mock.Mock(monotonic=mock.Mock(side_effect=[0, 0, 11]))) - with pytest.raises(RuntimeError, match="proxy on port 1081 did not start"): - setup_tests.setup_socks5() - for process in (first, second): - process.terminate.assert_called_once_with() - process.wait.assert_called_once_with(timeout=10) - write_env.assert_not_called() - - -def test_setup_rejects_exited_proxy(proxy_setup): - first, _second, popen, _connect, write_env = proxy_setup - first.poll.return_value = 1 - with pytest.raises(RuntimeError, match="proxy on port 1080 exited"): - setup_tests.setup_socks5() - popen.assert_called_once() - first.terminate.assert_not_called() - first.wait.assert_called_once_with(timeout=10) - write_env.assert_not_called() - - -@pytest.fixture -def proxy_teardown(monkeypatch): - monkeypatch.setenv("TEST_NAME", "socks5") - monkeypatch.setenv("SOCKS5_PROXY_PIDS", "11 22") - monkeypatch.setenv("COVERAGE", "") - script = str((Path(setup_tests.DRIVERS_TOOLS) / ".evergreen" / "socks5srv.py").absolute()) - result = subprocess.CompletedProcess([], 0, stdout=f"{sys.executable} {script} --port 1080") - with ( - mock.patch("shutil.which", return_value="/usr/bin/ps"), - mock.patch.object(subprocess, "run", return_value=result) as run, - mock.patch.object(os, "kill") as kill, - ): - yield result, run, kill - - -def test_teardown_stops_recorded_proxies(proxy_teardown): - _result, _run, kill = proxy_teardown - runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) - assert kill.call_args_list == [mock.call(11, signal.SIGTERM), mock.call(22, signal.SIGTERM)] - - -def test_teardown_refuses_reused_pid(proxy_teardown): - result, _run, kill = proxy_teardown - result.stdout = "python unrelated.py" - with pytest.raises(RuntimeError, match="Refusing to stop unrelated process"): - runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) - kill.assert_not_called() - - -@pytest.mark.parametrize("pid", [-1, 0]) -def test_teardown_rejects_invalid_pid(pid, proxy_teardown, monkeypatch): - _result, _run, kill = proxy_teardown - monkeypatch.setenv("SOCKS5_PROXY_PIDS", str(pid)) - with pytest.raises(ValueError, match="Invalid SOCKS5 proxy PID"): - runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) - kill.assert_not_called() - - -def test_teardown_accepts_exited_proxies(proxy_teardown): - result, _run, kill = proxy_teardown - result.returncode = 1 - runpy.run_path(str(setup_tests.HERE / "teardown_tests.py")) - kill.assert_not_called() - - -def test_runner_selects_sync_and_async_prose_tests(monkeypatch): - assert TEST_SUITE_MAP["socks5"] == "" - monkeypatch.setattr(run_tests, "TEST_NAME", "socks5") - monkeypatch.setattr(run_tests, "TEST_ARGS", ["-v"]) - monkeypatch.setattr(run_tests, "GREEN_FRAMEWORK", None) - monkeypatch.setattr(run_tests, "TEST_PERF", None) - monkeypatch.setattr(sys, "argv", ["run_tests.py"]) - monkeypatch.setenv("NO_EXT", "1") - monkeypatch.setenv("PYMONGOCRYPT_LIB", "") - monkeypatch.setenv("COVERAGE", "") - monkeypatch.setenv("DEBUG_LOG", "") - with ( - mock.patch.object(run_tests, "list_packages"), - mock.patch.object(run_tests.pytest, "main", return_value=0) as pytest_main, - ): - run_tests.run() - pytest_main.assert_called_once_with( - ["-v", "test/test_socks5.py", "test/asynchronous/test_socks5.py", "-k", "TestSocks5Prose"] - ) - - -@pytest.mark.parametrize( - ("uri", "expected"), - [ - ("mongodb://localhost/", "localhost:27017"), - ("mongodb://localhost:27018,localhost:27019/", "localhost:27018"), - ("mongodb://user:pass@localhost:27018/", "localhost:27018"), - ("mongodb://[::1]:27018,[::1]:27019/", "[::1]:27018"), - ], -) -def test_first_seed_mapping(uri, expected): - assert setup_tests._first_seed(uri) == expected diff --git a/pymongo/asynchronous/socks5_support.py b/pymongo/asynchronous/socks5_support.py index 4c6038df7e..90c2ca3c27 100644 --- a/pymongo/asynchronous/socks5_support.py +++ b/pymongo/asynchronous/socks5_support.py @@ -22,11 +22,7 @@ from pymongo.pool_shared import _configure_tcp_socket, _new_tcp_socket from pymongo.socks5_shared import ( _SOCKS5_ATYP_DOMAIN, - _SOCKS5_AUTH_REPLY_LEN, _SOCKS5_AUTH_USER_PASS, - _SOCKS5_CONNECT_REPLY_LEN, - _SOCKS5_DOMAIN_LENGTH_LEN, - _SOCKS5_GREETING_REPLY_LEN, _async_proxy_connect, _async_proxy_getaddrinfo, _async_socks5_recv, @@ -70,21 +66,15 @@ async def _async_socks5_handshake( greeting, allowed = _socks5_greeting(proxy) await _async_socks5_sendall(sock, greeting, deadline) method = _socks5_check_greeting_reply( - await _async_socks5_read_exact(sock, _SOCKS5_GREETING_REPLY_LEN, deadline), allowed + await _async_socks5_read_exact(sock, 2, deadline), allowed ) if method == _SOCKS5_AUTH_USER_PASS: await _async_socks5_sendall(sock, _socks5_auth_request(proxy), deadline) - _socks5_check_auth_reply( - await _async_socks5_read_exact(sock, _SOCKS5_AUTH_REPLY_LEN, deadline) - ) + _socks5_check_auth_reply(await _async_socks5_read_exact(sock, 2, deadline)) await _async_socks5_sendall(sock, _socks5_request(address), deadline) - atyp = _socks5_check_connect_reply( - await _async_socks5_read_exact(sock, _SOCKS5_CONNECT_REPLY_LEN, deadline) - ) + atyp = _socks5_check_connect_reply(await _async_socks5_read_exact(sock, 4, deadline)) domain_length = ( - (await _async_socks5_read_exact(sock, _SOCKS5_DOMAIN_LENGTH_LEN, deadline))[0] - if atyp == _SOCKS5_ATYP_DOMAIN - else None + (await _async_socks5_read_exact(sock, 1, deadline))[0] if atyp == _SOCKS5_ATYP_DOMAIN else 0 ) bound_size = _socks5_bound_address_size(atyp, domain_length) await _async_socks5_read_exact(sock, bound_size, deadline) diff --git a/pymongo/socks5_shared.py b/pymongo/socks5_shared.py index 390400752c..f5b4991d44 100644 --- a/pymongo/socks5_shared.py +++ b/pymongo/socks5_shared.py @@ -20,7 +20,7 @@ import ipaddress import socket import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, cast from pymongo import _csot from pymongo.asynchronous.helpers import _getaddrinfo @@ -37,8 +37,6 @@ _SOCKS5_AUTH_NO_ACCEPTABLE = 0xFF _SOCKS5_AUTH_SUCCESS = 0x00 _SOCKS5_RESERVED = 0x00 -_SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS = 1 -_SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS = 2 _SOCKS5_CMD_CONNECT = 0x01 _SOCKS5_REPLY_SUCCESS = 0x00 _SOCKS5_REPLY_GENERAL_FAILURE = 0x01 @@ -52,13 +50,6 @@ _SOCKS5_ATYP_IPV4 = 0x01 _SOCKS5_ATYP_DOMAIN = 0x03 _SOCKS5_ATYP_IPV6 = 0x04 -_SOCKS5_IPV4_BOUND_LEN = 6 -_SOCKS5_IPV6_BOUND_LEN = 18 -_SOCKS5_PORT_LEN = 2 -_SOCKS5_GREETING_REPLY_LEN = 2 -_SOCKS5_AUTH_REPLY_LEN = 2 -_SOCKS5_CONNECT_REPLY_LEN = 4 -_SOCKS5_DOMAIN_LENGTH_LEN = 1 _SOCKS5_REPLY_REASONS = { _SOCKS5_REPLY_GENERAL_FAILURE: "general SOCKS server failure", @@ -108,14 +99,12 @@ def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: return remaining -def _socks5_bound_address_size(atyp: int, domain_length: Optional[int] = None) -> int: +def _socks5_bound_address_size(atyp: int, domain_length: int = 0) -> int: if atyp == _SOCKS5_ATYP_IPV4: - return _SOCKS5_IPV4_BOUND_LEN + return 6 if atyp == _SOCKS5_ATYP_IPV6: - return _SOCKS5_IPV6_BOUND_LEN - if domain_length is None: - raise _Socks5Error("SOCKS5 connect failed: missing bound domain length") - return domain_length + _SOCKS5_PORT_LEN + return 18 + return domain_length + 2 def _socks5_greeting(proxy: _ProxyOptions) -> tuple[bytes, tuple[int, ...]]: @@ -123,7 +112,7 @@ def _socks5_greeting(proxy: _ProxyOptions) -> tuple[bytes, tuple[int, ...]]: return bytes( ( _SOCKS5_VERSION, - _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, + 2, _SOCKS5_AUTH_NO_AUTH, _SOCKS5_AUTH_USER_PASS, ) @@ -131,9 +120,7 @@ def _socks5_greeting(proxy: _ProxyOptions) -> tuple[bytes, tuple[int, ...]]: _SOCKS5_AUTH_NO_AUTH, _SOCKS5_AUTH_USER_PASS, ) - return bytes( - (_SOCKS5_VERSION, _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, _SOCKS5_AUTH_NO_AUTH) - ), (_SOCKS5_AUTH_NO_AUTH,) + return bytes((_SOCKS5_VERSION, 1, _SOCKS5_AUTH_NO_AUTH)), (_SOCKS5_AUTH_NO_AUTH,) def _socks5_check_greeting_reply(data: bytes, allowed: tuple[int, ...]) -> int: @@ -169,8 +156,7 @@ def _socks5_check_auth_reply(data: bytes) -> None: def _socks5_request(address: _Address) -> bytes: host, port = address - if port is None: - raise _Socks5Error("SOCKS5 connect failed: destination port is required") + port = cast(int, port) try: ip = ipaddress.ip_address(host.split("%", 1)[0]) except ValueError: @@ -188,7 +174,7 @@ def _socks5_request(address: _Address) -> bytes: return ( bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) + encoded_host - + port.to_bytes(_SOCKS5_PORT_LEN, "big") + + port.to_bytes(2, "big") ) diff --git a/pymongo/synchronous/socks5_support.py b/pymongo/synchronous/socks5_support.py index 6853294fef..9a9662e29b 100644 --- a/pymongo/synchronous/socks5_support.py +++ b/pymongo/synchronous/socks5_support.py @@ -22,11 +22,7 @@ from pymongo.pool_shared import _configure_tcp_socket, _new_tcp_socket from pymongo.socks5_shared import ( _SOCKS5_ATYP_DOMAIN, - _SOCKS5_AUTH_REPLY_LEN, _SOCKS5_AUTH_USER_PASS, - _SOCKS5_CONNECT_REPLY_LEN, - _SOCKS5_DOMAIN_LENGTH_LEN, - _SOCKS5_GREETING_REPLY_LEN, _proxy_connect, _proxy_deadline, _proxy_getaddrinfo, @@ -67,21 +63,13 @@ def _socks5_handshake( ) -> None: greeting, allowed = _socks5_greeting(proxy) _socks5_sendall(sock, greeting, deadline) - method = _socks5_check_greeting_reply( - _socks5_read_exact(sock, _SOCKS5_GREETING_REPLY_LEN, deadline), allowed - ) + method = _socks5_check_greeting_reply(_socks5_read_exact(sock, 2, deadline), allowed) if method == _SOCKS5_AUTH_USER_PASS: _socks5_sendall(sock, _socks5_auth_request(proxy), deadline) - _socks5_check_auth_reply(_socks5_read_exact(sock, _SOCKS5_AUTH_REPLY_LEN, deadline)) + _socks5_check_auth_reply(_socks5_read_exact(sock, 2, deadline)) _socks5_sendall(sock, _socks5_request(address), deadline) - atyp = _socks5_check_connect_reply( - _socks5_read_exact(sock, _SOCKS5_CONNECT_REPLY_LEN, deadline) - ) - domain_length = ( - (_socks5_read_exact(sock, _SOCKS5_DOMAIN_LENGTH_LEN, deadline))[0] - if atyp == _SOCKS5_ATYP_DOMAIN - else None - ) + atyp = _socks5_check_connect_reply(_socks5_read_exact(sock, 4, deadline)) + domain_length = (_socks5_read_exact(sock, 1, deadline))[0] if atyp == _SOCKS5_ATYP_DOMAIN else 0 bound_size = _socks5_bound_address_size(atyp, domain_length) _socks5_read_exact(sock, bound_size, deadline) diff --git a/test/asynchronous/test_socks5.py b/test/asynchronous/test_socks5.py deleted file mode 100644 index 9ab3288477..0000000000 --- a/test/asynchronous/test_socks5.py +++ /dev/null @@ -1,751 +0,0 @@ -# Copyright 2026-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for SOCKS5 proxy connection establishment.""" - -from __future__ import annotations - -import asyncio -import os -import socket -import ssl -import sys -from collections.abc import Iterator -from contextlib import contextmanager -from typing import Any, NamedTuple, Optional, cast -from unittest import mock -from urllib.parse import quote_plus - -import pytest - -sys.path[0:0] = [""] - -from pymongo import monitoring -from pymongo.hello import HelloCompat -from pymongo.pool_options import PoolOptions, _ProxyOptions -from pymongo.pool_shared import _async_configured_socket, _async_create_connection -from pymongo.socks5_shared import ( - _SOCKS5_ATYP_DOMAIN, - _SOCKS5_ATYP_IPV4, - _SOCKS5_ATYP_IPV6, - _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, - _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, - _SOCKS5_AUTH_NO_ACCEPTABLE, - _SOCKS5_AUTH_NO_AUTH, - _SOCKS5_AUTH_USER_PASS, - _SOCKS5_AUTH_VERSION, - _SOCKS5_CMD_CONNECT, - _SOCKS5_RESERVED, - _SOCKS5_VERSION, - _Socks5Error, - _Socks5Timeout, -) -from test.asynchronous import ( - AsyncIntegrationTest, - AsyncPyMongoTestCase, - async_client_context, - unittest, -) -from test.asynchronous.utils import async_wait_until -from test.helpers_shared import TLS_OPTIONS -from test.socks5_helpers import _Socks5Server, _TLSServer - -_IS_SYNC = False -pytestmark = pytest.mark.default_async - -_SOCKS5_AUTH_GSSAPI_METHOD = 0x01 -_SOCKS5_AUTH_FAILURE_STATUS = 0x01 -_SOCKS4_VERSION = 0x04 - - -class _ProseCase(NamedTuple): - mapped: bool - proxy_port: int - credentials: Optional[tuple[str, str]] - succeeds: bool - - -@contextmanager -def _fail_connection( - address: tuple[str, int], error_type: type[OSError] = ConnectionRefusedError -) -> Iterator[None]: - if _IS_SYNC: - original_connect = socket.socket.connect - - def connect(sock: socket.socket, target: tuple[str, int]) -> None: - if target == address: - raise error_type("connection failed") - original_connect(sock, target) - - with mock.patch.object(socket.socket, "connect", new=connect): - yield - else: - loop = asyncio.get_running_loop() - original_sock_connect = loop.sock_connect - - async def sock_connect(sock: socket.socket, target: tuple[str, int]) -> None: - if target == address: - raise error_type("connection failed") - await original_sock_connect(sock, target) - - with mock.patch.object(loop, "sock_connect", new=sock_connect): - yield - - -class TestSocks5(AsyncPyMongoTestCase): - def _create_server(self, **kwargs: Any) -> _Socks5Server: - server = _Socks5Server(**kwargs) - self.addAsyncCleanup(server.close) - return server - - async def _connect( - self, - server: _Socks5Server, - address: tuple[str, int], - username: Optional[str] = None, - password: Optional[str] = None, - timeout: float = 2, - ) -> socket.socket: - sock = await _async_create_connection( - address, - PoolOptions( - connect_timeout=timeout, - proxy_options=_ProxyOptions( - server.address[0], server.address[1], username, password - ), - ), - ) - self.addAsyncCleanup(sock.close) - return sock - - async def test_greeting_and_authenticated_connect(self) -> None: - server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) - sock = await self._connect(server, ("mongodb.example", 27017), "üser", "påss") - sock.close() - server.close() - self.assertEqual( - server.greeting, - bytes( - ( - _SOCKS5_VERSION, - _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, - _SOCKS5_AUTH_NO_AUTH, - _SOCKS5_AUTH_USER_PASS, - ) - ), - ) - self.assertEqual( - server.auth, - bytes((_SOCKS5_AUTH_VERSION, len("üser".encode()))) - + "üser".encode() - + bytes((len("påss".encode()),)) - + "påss".encode(), - ) - self.assertEqual( - server.request[:4], - bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED, _SOCKS5_ATYP_DOMAIN)), - ) - self.assertEqual( - server.request[4:], - bytes((len(b"mongodb.example"),)) + b"mongodb.example" + (27017).to_bytes(2, "big"), - ) - - async def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: - server = self._create_server(method=_SOCKS5_AUTH_NO_AUTH) - sock = await self._connect(server, ("127.0.0.1", 27017), "user", "password") - sock.close() - server.close() - self.assertEqual( - server.greeting, - bytes( - ( - _SOCKS5_VERSION, - _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, - _SOCKS5_AUTH_NO_AUTH, - _SOCKS5_AUTH_USER_PASS, - ) - ), - ) - self.assertEqual(server.auth, b"") - self.assertEqual( - server.request, - bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED, _SOCKS5_ATYP_IPV4)) - + socket.inet_aton("127.0.0.1") - + (27017).to_bytes(2, "big"), - ) - - async def test_no_credentials_only_offer_no_authentication(self) -> None: - server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) - with self.assertRaises(_Socks5Error): - await self._connect(server, ("localhost", 27017)) - server.close() - self.assertEqual( - server.greeting, - bytes( - (_SOCKS5_VERSION, _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, _SOCKS5_AUTH_NO_AUTH) - ), - ) - - async def test_rejects_all_unacceptable_methods(self) -> None: - for method in (_SOCKS5_AUTH_GSSAPI_METHOD, _SOCKS5_AUTH_NO_ACCEPTABLE): - server = self._create_server(method=method) - with self.subTest(method=method), self.assertRaises(_Socks5Error): - await self._connect(server, ("localhost", 27017)) - server.close() - - async def test_rejects_authentication_failure(self) -> None: - server = self._create_server( - method=_SOCKS5_AUTH_USER_PASS, auth_status=_SOCKS5_AUTH_FAILURE_STATUS - ) - with self.assertRaises(_Socks5Error): - await self._connect(server, ("localhost", 27017), "user", "password") - server.close() - self.assertEqual( - server.auth, - bytes((_SOCKS5_AUTH_VERSION, len(b"user"))) - + b"user" - + bytes((len(b"password"),)) - + b"password", - ) - - async def test_connect_request_address_types(self) -> None: - cases = ( - ("192.0.2.1", bytes((_SOCKS5_ATYP_IPV4,)) + socket.inet_aton("192.0.2.1")), - ( - "2001:db8::1", - bytes((_SOCKS5_ATYP_IPV6,)) + socket.inet_pton(socket.AF_INET6, "2001:db8::1"), - ), - ( - "münich.example", - bytes((_SOCKS5_ATYP_DOMAIN, len("xn--mnich-kva.example"))) - + "xn--mnich-kva.example".encode("ascii"), - ), - ) - for host, expected in cases: - server = self._create_server() - sock = await self._connect(server, (host, 27017)) - sock.close() - server.close() - self.assertEqual( - server.request, - bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) - + expected - + (27017).to_bytes(2, "big"), - ) - - async def test_drains_bound_address_and_preserves_early_payload(self) -> None: - for atyp in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): - server = self._create_server(bound_atyp=atyp, payload=b"mongo") - sock = await self._connect(server, ("localhost", 27017)) - if _IS_SYNC: - payload = sock.recv(5) - else: - sock.setblocking(False) - payload = await asyncio.get_running_loop().sock_recv(sock, 5) - sock.close() - server.close() - self.assertEqual(payload, b"mongo") - - async def test_rejects_socks_reply_codes(self) -> None: - for reply in range(1, 9): - server = self._create_server(reply=reply) - with self.subTest(reply=reply), self.assertRaises(_Socks5Error): - await self._connect(server, ("localhost", 27017)) - server.close() - - async def test_proxy_options_reach_server_and_monitor_pools(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - from pymongo.asynchronous.topology import Topology - - client = AsyncMongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) - try: - topology = Topology(client._topology_settings) - server_pool = topology._create_pool_for_server(("localhost", 27017)) - monitor_pool = topology._create_pool_for_monitor(("localhost", 27017)) - expected = _ProxyOptions("127.0.0.1") - self.assertEqual(server_pool.opts._proxy_options, expected) - self.assertEqual(monitor_pool.opts._proxy_options, expected) - finally: - await client.close() - - async def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - client = AsyncMongoClient( - "mongodb://localhost/?proxyHost=localhost&proxyPort=1234" - "&proxyUsername=uri-user&proxyPassword=uri-password", - proxyHost="[::1]", - proxyPort=5678, - proxyUsername="", - proxyPassword="", - connect=False, - ) - try: - self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1", 5678)) - finally: - await client.close() - - async def test_client_equality_includes_proxy_options(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - clients = [ - AsyncMongoClient("mongodb://localhost/?proxyHost=proxy.example", connect=False), - AsyncMongoClient("mongodb://localhost/?proxyHost=proxy.example", connect=False), - AsyncMongoClient("mongodb://localhost/?proxyHost=other.example", connect=False), - ] - try: - self.assertEqual(clients[0], clients[1]) - self.assertNotEqual(clients[0], clients[2]) - finally: - for client in clients: - await client.close() - - async def test_bad_versions_and_unexpected_eof_fail(self) -> None: - cases = ( - (self._create_server(method_version=_SOCKS4_VERSION), ("localhost", 27017), None), - (self._create_server(reply_version=_SOCKS4_VERSION), ("localhost", 27017), None), - (self._create_server(eof_stage="greeting"), ("localhost", 27017), None), - ( - self._create_server(method=_SOCKS5_AUTH_USER_PASS, eof_stage="auth"), - ("localhost", 27017), - ("user", "password"), - ), - (self._create_server(eof_stage="connect"), ("localhost", 27017), None), - ) - for server, address, credentials in cases: - with self.subTest(stage=server.eof_stage, version=server.method_version): - with self.assertRaises(_Socks5Error): - await self._connect(server, address, *(credentials or (None, None))) - server.close() - - async def test_timeout_covers_the_entire_handshake(self) -> None: - server = self._create_server(stall=True) - with self.assertRaisesRegex(socket.timeout, "SOCKS5 proxy timed out"): - await self._connect(server, ("localhost", 27017), timeout=0.1) - server.close() - - @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") - async def test_cancellation_closes_proxy_socket(self) -> None: - server = self._create_server(stall=True) - task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) - await async_wait_until(server.started.is_set, "receive the SOCKS5 greeting", timeout=2) - task.cancel() - with self.assertRaises(asyncio.CancelledError): - await task - - @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") - async def test_cancellation_when_proxy_send_completes_closes_socket(self) -> None: - server = self._create_server(stall=True) - loop = asyncio.get_running_loop() - original_sendall = loop.sock_sendall - sockets: list[socket.socket] = [] - - async def cancel_after_send(sock: socket.socket, data: bytes) -> None: - sockets.append(sock) - await original_sendall(sock, data) - task.cancel() - - with mock.patch.object(loop, "sock_sendall", new=cancel_after_send): - task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) - with self.assertRaises(asyncio.CancelledError): - await task - self.assertEqual(len(sockets), 1) - self.assertEqual(sockets[0].fileno(), -1) - - async def test_tls_sni_targets_destination_not_proxy(self) -> None: - sni_names: list[Optional[str]] = [] - tls_server = _TLSServer(sni_names) - self.addAsyncCleanup(tls_server.close) - proxy = self._create_server(target=tls_server.address) - client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - client_context.check_hostname = False - client_context.verify_mode = ssl.CERT_NONE - sock = await _async_configured_socket( - ("mongodb.example", tls_server.address[1]), - PoolOptions( - connect_timeout=2, - ssl_context=cast(Any, client_context), - proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), - ), - ) - self.addAsyncCleanup(sock.close) - sock.close() - self.assertEqual(sni_names, ["mongodb.example"]) - - async def test_proxy_password_is_redacted_in_repr(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - for client in ( - AsyncMongoClient( - "mongodb://user:dbsecret@localhost/?proxyHost=127.0.0.1" - "&proxyUsername=proxyuser&proxyPassword=hunter2", - connect=False, - ), - AsyncMongoClient( - "mongodb://localhost/", - username="user", - password="dbsecret", - proxyHost="127.0.0.1", - proxyUsername="proxyuser", - proxyPassword="hunter2", - connect=False, - ), - ): - try: - client_repr = repr(client) - proxy_repr = repr(client.options.pool_options._proxy_options) - self.assertNotIn("hunter2", client_repr) - self.assertNotIn("proxyuser", client_repr) - self.assertNotIn("dbsecret", client_repr) - self.assertNotIn("hunter2", proxy_repr) - self.assertIn("127.0.0.1", proxy_repr) - finally: - await client.close() - - async def test_split_proxy_options_across_uri_and_kwargs(self) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - client1 = AsyncMongoClient( - "mongodb://localhost/?proxyPort=1081", - proxyHost="proxy.example", - connect=False, - ) - client2 = AsyncMongoClient( - "mongodb://localhost/?proxyHost=proxy.example&proxyUsername=u", - proxyPassword="p", - connect=False, - ) - try: - self.assertEqual( - client1.options.pool_options._proxy_options, - _ProxyOptions("proxy.example", 1081), - ) - self.assertEqual( - client2.options.pool_options._proxy_options, - _ProxyOptions("proxy.example", 1080, "u", "p"), - ) - finally: - await client1.close() - await client2.close() - - async def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: - from pymongo.asynchronous.pool import Pool - from pymongo.errors import AutoReconnect, NetworkTimeout - from pymongo.pool_shared import _raise_connection_failure - from pymongo.socks5_shared import _Socks5Timeout - - pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) - self.addAsyncCleanup(pool.close) - for cause, overloaded in ( - (_Socks5Error("boom"), False), - (_Socks5Timeout("SOCKS5 proxy timed out"), False), - (OSError("boom"), True), - ): - with self.subTest(cause=type(cause).__name__): - with self.assertRaises((AutoReconnect, NetworkTimeout)) as raised: - _raise_connection_failure(("localhost", 27017), cause) - wrapped = raised.exception - pool._handle_connection_error(wrapped) - self.assertEqual(wrapped.has_error_label("SystemOverloadedError"), overloaded) - self.assertEqual(wrapped.has_error_label("RetryableError"), overloaded) - - async def test_multi_address_proxy_fallback_on_connect_error(self) -> None: - dead_addr = ("127.0.0.1", 1) - target = ( - "pymongo.socks5_shared.socket.getaddrinfo" - if _IS_SYNC - else "pymongo.socks5_shared._getaddrinfo" - ) - for error_type in (ConnectionRefusedError, socket.timeout): - server = self._create_server() - fake_results = [ - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), - ] - - async def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: - return fake_results - - replacement: Any = ( - (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo - ) - with ( - self.subTest(error_type=error_type.__name__), - mock.patch(target, new=replacement), - _fail_connection(dead_addr, error_type), - ): - sock = await self._connect(server, ("localhost", 27017)) - sock.close() - - async def test_proxy_connect_timeout_does_not_retry_after_deadline(self) -> None: - import pymongo.asynchronous.socks5_support as socks5_module - from pymongo.socks5_shared import _Socks5Timeout - - addresses = [ - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)) - for port in (1, 2) - ] - with ( - mock.patch.object(socks5_module, "_proxy_deadline", return_value=0), - mock.patch.object(socks5_module, "_async_proxy_getaddrinfo", return_value=addresses), - mock.patch.object( - socks5_module, "_async_proxy_connect", side_effect=socket.timeout("timed out") - ) as connect, - self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out") as raised, - ): - await _async_create_connection( - ("mongodb.example", 27017), - PoolOptions(proxy_options=_ProxyOptions("proxy.example")), - ) - connect.assert_called_once() - self.assertEqual(connect.call_args.args[0].fileno(), -1) - self.assertIsInstance(raised.exception.__cause__, socket.timeout) - - async def test_proxy_connection_error_paths(self) -> None: - proxy_opts = PoolOptions( - connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080) - ) - target = ( - "pymongo.socks5_shared.socket.getaddrinfo" - if _IS_SYNC - else "pymongo.socks5_shared._getaddrinfo" - ) - - def raise_gai(*_a: Any, **_kw: Any) -> Any: - raise socket.gaierror("resolution failed") - - with ( - mock.patch(target, side_effect=raise_gai), - self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), - ): - await _async_create_connection(("localhost", 27017), proxy_opts) - - with ( - mock.patch(target, side_effect=socket.timeout("timed out")), - self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), - ): - await _async_create_connection(("localhost", 27017), proxy_opts) - - async def empty_async(*_a: Any, **_kw: Any) -> Any: - return [] - - replacement_empty: Any = (lambda *_a, **_kw: []) if _IS_SYNC else empty_async - with ( - mock.patch(target, new=replacement_empty), - self.assertRaisesRegex(_Socks5Error, "returned no addresses"), - ): - await _async_create_connection(("localhost", 27017), proxy_opts) - - dead_addr = ("127.0.0.1", 1) - dead_results = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr)] - - async def dead_async(*_a: Any, **_kw: Any) -> Any: - return dead_results - - replacement_dead: Any = (lambda *_a, **_kw: dead_results) if _IS_SYNC else dead_async - with ( - mock.patch(target, new=replacement_dead), - _fail_connection(dead_addr), - self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), - ): - await _async_create_connection(("localhost", 27017), proxy_opts) - - with ( - mock.patch(target, new=replacement_dead), - _fail_connection(dead_addr, socket.timeout), - self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), - ): - await _async_create_connection(("localhost", 27017), proxy_opts) - - server = self._create_server() - handshake_options = PoolOptions( - connect_timeout=1, - proxy_options=_ProxyOptions(server.address[0], server.address[1]), - ) - sendall_target = ( - "pymongo.synchronous.socks5_support._socks5_sendall" - if _IS_SYNC - else "pymongo.asynchronous.socks5_support._async_socks5_sendall" - ) - with ( - mock.patch(sendall_target, side_effect=OSError("handshake failed")), - self.assertRaisesRegex(_Socks5Error, "SOCKS5 handshake failed"), - ): - await _async_create_connection(("localhost", 27017), handshake_options) - server.close() - - server = self._create_server() - handshake_options = PoolOptions( - connect_timeout=1, - proxy_options=_ProxyOptions(server.address[0], server.address[1]), - ) - with ( - mock.patch(sendall_target, side_effect=KeyboardInterrupt), - self.assertRaises(KeyboardInterrupt), - ): - await _async_create_connection(("localhost", 27017), handshake_options) - server.close() - - -class _CommandListener(monitoring.CommandListener): - def __init__(self) -> None: - self.connection_ids: list[tuple[str, Optional[int]]] = [] - - def started(self, event: monitoring.CommandStartedEvent) -> None: - self.connection_ids.append(event.connection_id) - - def succeeded(self, event: monitoring.CommandSucceededEvent) -> None: - self.connection_ids.append(event.connection_id) - - def failed(self, event: monitoring.CommandFailedEvent) -> None: - self.connection_ids.append(event.connection_id) - - -class TestSocks5Prose(AsyncIntegrationTest): - @async_client_context.require_replica_set - async def asyncSetUp(self) -> None: - if not os.environ.get("SOCKS5_PROXIES_STARTED"): - self.skipTest("SOCKS5 proxies must be started by the Evergreen SOCKS5 tasks.") - await super().asyncSetUp() - self.nodes = sorted(async_client_context.nodes) - self.auth_port = 1080 - self.noauth_port = 1081 - - def _hosts(self, mapped: bool) -> str: - if mapped: - return "localhost:12345" - return ",".join( - f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" - for host, port in self.nodes - ) - - async def _run_case( - self, - hosts: str, - port: int, - credentials: Optional[tuple[str, str]], - succeeds: bool, - use_uri: bool, - listener: Optional[_CommandListener] = None, - ) -> None: - from pymongo.asynchronous.mongo_client import AsyncMongoClient - - proxy_options = {"proxyHost": "localhost", "proxyPort": port} - if credentials: - proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials - tls_options = dict(TLS_OPTIONS) if async_client_context.tls else {} - options = dict(tls_options) - options.update(self.credentials) - options["serverSelectionTimeoutMS"] = 2500 - if hosts == "localhost:12345": - options["directConnection"] = True - if listener is not None: - options["event_listeners"] = [listener] - if use_uri: - uri_options = "&".join( - f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() - ) - hosts = f"mongodb://{hosts}/?{uri_options}" - else: - options.update(proxy_options) - client = AsyncMongoClient(hosts, **options) - try: - if succeeds: - await client.admin.command(HelloCompat.CMD) - if listener is not None: - from pymongo.errors import OperationFailure - - with self.assertRaises(OperationFailure): - await client.admin.command("nonexistent_socks5_test_command") - else: - from pymongo.errors import ConnectionFailure - - with self.assertRaises(ConnectionFailure): - await client.admin.command(HelloCompat.CMD) - finally: - await client.close() - - async def test_connection_string_prose_table(self) -> None: - cases = ( - _ProseCase(mapped=True, proxy_port=self.auth_port, credentials=None, succeeds=False), - _ProseCase(mapped=True, proxy_port=self.noauth_port, credentials=None, succeeds=True), - _ProseCase(mapped=False, proxy_port=self.auth_port, credentials=None, succeeds=False), - _ProseCase(mapped=False, proxy_port=self.noauth_port, credentials=None, succeeds=True), - _ProseCase( - mapped=True, - proxy_port=self.auth_port, - credentials=("nonexistentuser", "badauth"), - succeeds=False, - ), - _ProseCase( - mapped=True, - proxy_port=self.noauth_port, - credentials=("nonexistentuser", "badauth"), - succeeds=True, - ), - _ProseCase( - mapped=False, - proxy_port=self.noauth_port, - credentials=("nonexistentuser", "badauth"), - succeeds=True, - ), - _ProseCase( - mapped=True, - proxy_port=self.auth_port, - credentials=("username", "p4ssw0rd"), - succeeds=True, - ), - _ProseCase( - mapped=False, - proxy_port=self.auth_port, - credentials=("username", "p4ssw0rd"), - succeeds=True, - ), - ) - for case in cases: - for use_uri in (True, False): - with self.subTest( - mapped=case.mapped, - proxy_port=case.proxy_port, - credentials=case.credentials, - use_uri=use_uri, - ): - await self._run_case( - hosts=self._hosts(case.mapped), - port=case.proxy_port, - credentials=case.credentials, - succeeds=case.succeeds, - use_uri=use_uri, - ) - - async def test_command_monitoring_does_not_reference_proxy(self) -> None: - proxy_addresses = { - ("localhost", self.auth_port), - ("localhost", self.noauth_port), - ("127.0.0.1", self.auth_port), - ("127.0.0.1", self.noauth_port), - } - for mapped in (True, False): - listener = _CommandListener() - with self.subTest(mapped=mapped): - await self._run_case( - self._hosts(mapped), self.noauth_port, None, True, False, listener - ) - self.assertGreaterEqual(len(listener.connection_ids), 4) - for connection_id in listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - if mapped: - self.assertEqual(connection_id, ("localhost", 12345)) - else: - self.assertIn(connection_id, self.nodes) diff --git a/test/socks5_helpers.py b/test/socks5_helpers.py deleted file mode 100644 index 577afa7bb9..0000000000 --- a/test/socks5_helpers.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright 2026-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Threaded SOCKS5 test servers shared by synchronous and asynchronous tests.""" - -from __future__ import annotations - -import select -import socket -import ssl -import threading -from typing import Optional - -from pymongo.socks5_shared import ( - _SOCKS5_ATYP_DOMAIN, - _SOCKS5_ATYP_IPV4, - _SOCKS5_ATYP_IPV6, - _SOCKS5_AUTH_NO_AUTH, - _SOCKS5_AUTH_REPLY_LEN, - _SOCKS5_AUTH_SUCCESS, - _SOCKS5_AUTH_USER_PASS, - _SOCKS5_AUTH_VERSION, - _SOCKS5_CONNECT_REPLY_LEN, - _SOCKS5_DOMAIN_LENGTH_LEN, - _SOCKS5_GREETING_REPLY_LEN, - _SOCKS5_IPV4_BOUND_LEN, - _SOCKS5_IPV6_BOUND_LEN, - _SOCKS5_PORT_LEN, - _SOCKS5_REPLY_SUCCESS, - _SOCKS5_RESERVED, - _SOCKS5_VERSION, -) -from test.helpers_shared import CERT_PATH - - -class _TLSServer: - def __init__(self, sni_names: list[Optional[str]]) -> None: - self.sni_names = sni_names - self.listener = socket.socket() - self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - self.listener.bind(("127.0.0.1", 0)) - self.listener.listen(1) - self.listener.settimeout(3) - self.address = self.listener.getsockname() - self.thread = threading.Thread(target=self._serve, daemon=True) - self.thread.start() - - def _serve(self) -> None: - try: - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - context.load_cert_chain(f"{CERT_PATH}/server.pem") - context.set_servername_callback(lambda _sock, name, _ctx: self.sni_names.append(name)) - with self.listener.accept()[0] as raw: - with context.wrap_socket(raw, server_side=True) as conn: - conn.recv(1) - except OSError: - pass - finally: - self.listener.close() - - def close(self) -> None: - self.listener.close() - self.thread.join(timeout=3) - - -class _Socks5Server: - def __init__( - self, - method: int = _SOCKS5_AUTH_NO_AUTH, - auth_status: int = _SOCKS5_AUTH_SUCCESS, - reply: int = _SOCKS5_REPLY_SUCCESS, - bound_atyp: int = _SOCKS5_ATYP_IPV4, - payload: bytes = b"", - method_version: int = _SOCKS5_VERSION, - reply_version: int = _SOCKS5_VERSION, - eof_stage: Optional[str] = None, - stall: bool = False, - target: Optional[tuple[str, int]] = None, - ) -> None: - self.method = method - self.auth_status = auth_status - self.reply = reply - self.bound_atyp = bound_atyp - self.payload = payload - self.method_version = method_version - self.reply_version = reply_version - self.eof_stage = eof_stage - self.stall = stall - self.target = target - self.started = threading.Event() - self.greeting = b"" - self.auth = b"" - self.request = b"" - self.error: Optional[BaseException] = None - self.listener = socket.socket() - self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - self.listener.bind(("127.0.0.1", 0)) - self.listener.listen(1) - self.listener.settimeout(3) - self.address = self.listener.getsockname() - self.thread = threading.Thread(target=self._serve, daemon=True) - self.thread.start() - - @staticmethod - def _read(sock: socket.socket, size: int) -> bytes: - data = bytearray() - while len(data) < size: - part = sock.recv(size - len(data)) - if not part: - raise EOFError("client closed connection") - data.extend(part) - return bytes(data) - - def _serve(self) -> None: - upstream: Optional[socket.socket] = None - try: - with self.listener.accept()[0] as conn: - conn.settimeout(3) - version, nmethods = self._read(conn, _SOCKS5_GREETING_REPLY_LEN) - self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) - self.started.set() - if self.stall: - conn.recv(1) - return - if self.eof_stage == "greeting": - conn.sendall(bytes((_SOCKS5_VERSION,))) - return - conn.sendall(bytes((self.method_version, self.method))) - if self.method == _SOCKS5_AUTH_USER_PASS: - auth_version, username_length = self._read(conn, _SOCKS5_AUTH_REPLY_LEN) - username = self._read(conn, username_length) - password_length = self._read(conn, _SOCKS5_DOMAIN_LENGTH_LEN)[0] - password = self._read(conn, password_length) - self.auth = ( - bytes((auth_version, username_length)) - + username - + bytes((password_length,)) - + password - ) - if self.eof_stage == "auth": - conn.sendall(bytes((_SOCKS5_AUTH_VERSION,))) - return - conn.sendall(bytes((_SOCKS5_AUTH_VERSION, self.auth_status))) - if self.auth_status: - return - version, command, reserved, atyp = self._read(conn, _SOCKS5_CONNECT_REPLY_LEN) - if atyp == _SOCKS5_ATYP_IPV4: - addr = self._read(conn, _SOCKS5_IPV4_BOUND_LEN - _SOCKS5_PORT_LEN) - elif atyp == _SOCKS5_ATYP_IPV6: - addr = self._read(conn, _SOCKS5_IPV6_BOUND_LEN - _SOCKS5_PORT_LEN) - else: - length = self._read(conn, _SOCKS5_DOMAIN_LENGTH_LEN) - addr = length + self._read(conn, length[0]) - self.request = ( - bytes((version, command, reserved, atyp)) - + addr - + self._read(conn, _SOCKS5_PORT_LEN) - ) - if self.eof_stage == "connect": - conn.sendall(bytes((_SOCKS5_VERSION, _SOCKS5_REPLY_SUCCESS))) - return - if self.target is not None: - upstream = socket.create_connection(self.target, timeout=3) - if self.bound_atyp == _SOCKS5_ATYP_IPV4: - bound = ( - bytes((_SOCKS5_ATYP_IPV4,)) - + socket.inet_aton("127.0.0.1") - + bytes(_SOCKS5_PORT_LEN) - ) - elif self.bound_atyp == _SOCKS5_ATYP_IPV6: - bound = ( - bytes((_SOCKS5_ATYP_IPV6,)) - + socket.inet_pton(socket.AF_INET6, "::1") - + bytes(_SOCKS5_PORT_LEN) - ) - else: - host = b"x" - bound = bytes((_SOCKS5_ATYP_DOMAIN, len(host))) + host + bytes(_SOCKS5_PORT_LEN) - conn.sendall( - bytes((self.reply_version, self.reply, _SOCKS5_RESERVED)) + bound + self.payload - ) - if self.payload: - return - if upstream is not None: - conn.setblocking(False) - upstream.setblocking(False) - while True: - readable, _, _ = select.select([conn, upstream], [], [], 5) - if not readable: - return - for source in readable: - data = source.recv(65536) - if not data: - return - destination = upstream if source is conn else conn - destination.sendall(data) - else: - while conn.recv(1024): - pass - except BaseException as exc: - self.error = exc - finally: - if upstream is not None: - upstream.close() - self.listener.close() - - def close(self) -> None: - self.listener.close() - self.thread.join(timeout=3) diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index 29439ab636..887ff56888 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -1060,127 +1060,83 @@ async def test_datagram_socket_from_callback_is_rejected(api): @both_apis -async def test_kms_request_uses_proxy_options(api): - if not _HAVE_PYMONGOCRYPT: - pytest.skip("pymongocrypt is not installed") - encryption_module = api.encryption() - - class FakeOpts: - _kms_connect_callback = None - - @staticmethod - def _kms_ssl_contexts(_is_sync): - return {"aws": object()} - - client = api.simple_client("mongodb://localhost/", connect=False) +@pytest.mark.skipif(not _HAVE_PYMONGOCRYPT, reason="pymongocrypt is not installed") +async def test_socks5_proxy_options_flow_to_kms(api): + module = api.encryption() + proxy = _ProxyOptions("10.0.0.2") + client = api.simple_client("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + clients = [] + explicit = None + monkeypatch = pytest.MonkeyPatch() try: - for expected in (None, _ProxyOptions("127.0.0.1")): - with pytest.MonkeyPatch.context() as monkeypatch: - io = encryption_module._EncryptionIO( - client, - client.keyvault.datakeys, - None, - FakeOpts(), - proxy_options=expected, - ) - connection = mock.MagicMock() - - async def connect(_address, options, _callback, _timeout): - assert options._proxy_options is expected - return connection - - async def sendall(_connection, _message): - pass + opts = mock.Mock( + _kms_connect_callback=None, + _kms_ssl_contexts=mock.Mock(return_value={"aws": object()}), + ) + io = module._EncryptionIO(client, client.keyvault.datakeys, None, opts, proxy_options=proxy) + connection = mock.MagicMock() - if api.is_async: - monkeypatch.setattr(encryption_module, "_connect_kms", connect) - monkeypatch.setattr(encryption_module, "async_socket_sendall", sendall) - else: - - def sync_connect(_address, options, _callback, _timeout): - assert options._proxy_options is expected - return connection - - monkeypatch.setattr(encryption_module, "_connect_kms", sync_connect) - request = SimpleNamespace( - endpoint="kms.example:443", - message=b"request", - kms_provider="aws", - usleep=0, - bytes_needed=0, - ) - await api.maybe_await(io.kms_request(request)) - connection.close.assert_called_once_with() - finally: - await api.maybe_await(client.close()) + def connect(_address, options, _callback, _timeout): + assert options._proxy_options == proxy + return connection + patched_connect = ( + mock.AsyncMock(side_effect=connect) if api.is_async else mock.Mock(side_effect=connect) + ) + monkeypatch.setattr(module, "_connect_kms", patched_connect) + if api.is_async: + monkeypatch.setattr(module, "async_socket_sendall", mock.AsyncMock()) + request = SimpleNamespace( + endpoint="kms.example:443", + message=b"request", + kms_provider="aws", + usleep=0, + bytes_needed=0, + ) + await api.maybe_await(io.kms_request(request)) + connection.close.assert_called_once_with() -@both_apis -@pytest.mark.skipif(not _HAVE_PYMONGOCRYPT, reason="pymongocrypt is not installed") -async def test_auto_encryption_kms_proxy_uses_encrypted_client(api): - kv_client = api.simple_client("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - clients = [] - try: for bypass in (False, True): - client = api.simple_client( + encrypted = api.simple_client( "mongodb://localhost/?proxyHost=10.0.0.1", auto_encryption_opts=AutoEncryptionOpts( {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", - key_vault_client=kv_client, + key_vault_client=client, bypass_auto_encryption=bypass, ), connect=False, ) - clients.append(client) - assert client._encrypter is not None - io_callbacks = client._encrypter._auto_encrypter.callback - assert io_callbacks._proxy_options == _ProxyOptions("10.0.0.1") - assert (io_callbacks.client_ref is None) is bypass - assert io_callbacks.mongocryptd_client.options.pool_options._proxy_options is None - finally: - for client in clients: - await api.maybe_await(client.close()) - await api.maybe_await(kv_client.close()) - - -@both_apis -@pytest.mark.skipif(not _HAVE_PYMONGOCRYPT, reason="pymongocrypt is not installed") -async def test_explicit_encryption_kms_proxy_uses_key_vault_client(api): - kv_client = api.simple_client("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - encryption = api.client_encryption( - {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", kv_client, None - ) - try: - assert encryption._io_callbacks is not None - assert encryption._io_callbacks._proxy_options == _ProxyOptions("10.0.0.2") - finally: - await api.maybe_await(encryption.close()) - await api.maybe_await(kv_client.close()) - - -@both_apis -async def test_encrypter_rejects_callback_with_socks5_proxy(api): - callback = mock.Mock() - opts = AutoEncryptionOpts({}, "keyvault.datakeys", kms_connect_callback=callback) - client = mock.Mock( - options=mock.Mock(pool_options=PoolOptions(proxy_options=_ProxyOptions("proxy.example"))) - ) - with pytest.raises(ConfigurationError, match="kms_connect_callback with proxyHost"): - api.encryption()._Encrypter(client, opts) - - if _HAVE_PYMONGOCRYPT: - kv_client = api.simple_client("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) - try: + clients.append(encrypted) + callbacks = encrypted._encrypter._auto_encrypter.callback + assert callbacks._proxy_options == _ProxyOptions("10.0.0.1") + assert (callbacks.client_ref is None) is bypass + assert callbacks.mongocryptd_client.options.pool_options._proxy_options is None + + explicit = api.client_encryption( + {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", client, None + ) + assert explicit._io_callbacks._proxy_options == proxy + callback = api.callback(lambda _context: None) + opts = AutoEncryptionOpts({}, "keyvault.datakeys", kms_connect_callback=callback) + client_with_proxy = mock.Mock( + options=mock.Mock(pool_options=PoolOptions(proxy_options=proxy)) + ) + for create in ( + lambda: module._Encrypter(client_with_proxy, opts), + lambda: api.client_encryption( + {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", client, callback + ), + ): with pytest.raises(ConfigurationError, match="kms_connect_callback with proxyHost"): - api.client_encryption( - {"local": {"key": LOCAL_MASTER_KEY}}, - "keyvault.datakeys", - kv_client, - api.callback(callback), - ) - finally: - await api.maybe_await(kv_client.close()) + create() + finally: + monkeypatch.undo() + if explicit is not None: + await api.maybe_await(explicit.close()) + for encrypted in clients: + await api.maybe_await(encrypted.close()) + await api.maybe_await(client.close()) @both_apis diff --git a/test/test_socks5.py b/test/test_socks5.py index b6f4fc1215..bdbd41d3f8 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -18,11 +18,14 @@ import asyncio import os +import select import socket import ssl +import subprocess import sys -from collections.abc import Iterator -from contextlib import contextmanager +import threading +import time +from pathlib import Path from typing import Any, NamedTuple, Optional, cast from unittest import mock from urllib.parse import quote_plus @@ -34,13 +37,16 @@ from pymongo import monitoring from pymongo.hello import HelloCompat from pymongo.pool_options import PoolOptions, _ProxyOptions -from pymongo.pool_shared import _configured_socket, _create_connection +from pymongo.pool_shared import ( + _async_configured_socket, + _async_create_connection, + _configured_socket, + _create_connection, +) from pymongo.socks5_shared import ( _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_IPV6, - _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, - _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, _SOCKS5_AUTH_NO_ACCEPTABLE, _SOCKS5_AUTH_NO_AUTH, _SOCKS5_AUTH_USER_PASS, @@ -51,17 +57,11 @@ _Socks5Error, _Socks5Timeout, ) -from test import ( - IntegrationTest, - PyMongoTestCase, - client_context, - unittest, -) -from test.helpers_shared import TLS_OPTIONS -from test.socks5_helpers import _Socks5Server, _TLSServer -from test.utils import wait_until +from test import IntegrationTest, PyMongoTestCase, client_context +from test.asynchronous import AsyncIntegrationTest, AsyncPyMongoTestCase, async_client_context +from test.asynchronous.utils import async_wait_until +from test.helpers_shared import CERT_PATH, TLS_OPTIONS -_IS_SYNC = True pytestmark = pytest.mark.default _SOCKS5_AUTH_GSSAPI_METHOD = 0x01 @@ -76,33 +76,6 @@ class _ProseCase(NamedTuple): succeeds: bool -@contextmanager -def _fail_connection( - address: tuple[str, int], error_type: type[OSError] = ConnectionRefusedError -) -> Iterator[None]: - if _IS_SYNC: - original_connect = socket.socket.connect - - def connect(sock: socket.socket, target: tuple[str, int]) -> None: - if target == address: - raise error_type("connection failed") - original_connect(sock, target) - - with mock.patch.object(socket.socket, "connect", new=connect): - yield - else: - loop = asyncio.get_running_loop() - original_sock_connect = loop.sock_connect - - def sock_connect(sock: socket.socket, target: tuple[str, int]) -> None: - if target == address: - raise error_type("connection failed") - original_sock_connect(sock, target) - - with mock.patch.object(loop, "sock_connect", new=sock_connect): - yield - - class TestSocks5(PyMongoTestCase): def _create_server(self, **kwargs: Any) -> _Socks5Server: server = _Socks5Server(**kwargs) @@ -112,7 +85,7 @@ def _create_server(self, **kwargs: Any) -> _Socks5Server: def _connect( self, server: _Socks5Server, - address: tuple[str, int], + address: tuple[str, int] = ("localhost", 27017), username: Optional[str] = None, password: Optional[str] = None, timeout: float = 2, @@ -129,21 +102,13 @@ def _connect( self.addCleanup(sock.close) return sock - def test_greeting_and_authenticated_connect(self) -> None: + def test_greeting_and_auth(self) -> None: server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) - sock = self._connect(server, ("mongodb.example", 27017), "üser", "påss") - sock.close() + self._connect(server, ("mongodb.example", 27017), "üser", "påss").close() server.close() self.assertEqual( server.greeting, - bytes( - ( - _SOCKS5_VERSION, - _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, - _SOCKS5_AUTH_NO_AUTH, - _SOCKS5_AUTH_USER_PASS, - ) - ), + bytes((_SOCKS5_VERSION, 2, _SOCKS5_AUTH_NO_AUTH, _SOCKS5_AUTH_USER_PASS)), ) self.assertEqual( server.auth, @@ -153,74 +118,45 @@ def test_greeting_and_authenticated_connect(self) -> None: + "påss".encode(), ) self.assertEqual( - server.request[:4], - bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED, _SOCKS5_ATYP_DOMAIN)), - ) - self.assertEqual( - server.request[4:], - bytes((len(b"mongodb.example"),)) + b"mongodb.example" + (27017).to_bytes(2, "big"), - ) - - def test_no_auth_method_is_preferred_when_credentials_exist(self) -> None: - server = self._create_server(method=_SOCKS5_AUTH_NO_AUTH) - sock = self._connect(server, ("127.0.0.1", 27017), "user", "password") - sock.close() - server.close() - self.assertEqual( - server.greeting, + server.request, bytes( ( _SOCKS5_VERSION, - _SOCKS5_AUTH_METHOD_COUNT_WITH_CREDENTIALS, - _SOCKS5_AUTH_NO_AUTH, - _SOCKS5_AUTH_USER_PASS, + _SOCKS5_CMD_CONNECT, + _SOCKS5_RESERVED, + _SOCKS5_ATYP_DOMAIN, + len(b"mongodb.example"), ) - ), - ) - self.assertEqual(server.auth, b"") - self.assertEqual( - server.request, - bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED, _SOCKS5_ATYP_IPV4)) - + socket.inet_aton("127.0.0.1") + ) + + b"mongodb.example" + (27017).to_bytes(2, "big"), ) - def test_no_credentials_only_offer_no_authentication(self) -> None: - server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) - with self.assertRaises(_Socks5Error): - self._connect(server, ("localhost", 27017)) + server = self._create_server(method=_SOCKS5_AUTH_NO_AUTH) + self._connect(server, ("127.0.0.1", 27017), "user", "password").close() server.close() - self.assertEqual( - server.greeting, - bytes( - (_SOCKS5_VERSION, _SOCKS5_AUTH_METHOD_COUNT_NO_CREDENTIALS, _SOCKS5_AUTH_NO_AUTH) - ), - ) + self.assertEqual(server.auth, b"") - def test_rejects_all_unacceptable_methods(self) -> None: - for method in (_SOCKS5_AUTH_GSSAPI_METHOD, _SOCKS5_AUTH_NO_ACCEPTABLE): + for method in ( + _SOCKS5_AUTH_USER_PASS, + _SOCKS5_AUTH_GSSAPI_METHOD, + _SOCKS5_AUTH_NO_ACCEPTABLE, + ): server = self._create_server(method=method) with self.subTest(method=method), self.assertRaises(_Socks5Error): - self._connect(server, ("localhost", 27017)) + self._connect(server) server.close() + self.assertEqual(server.greeting, bytes((_SOCKS5_VERSION, 1, _SOCKS5_AUTH_NO_AUTH))) - def test_rejects_authentication_failure(self) -> None: server = self._create_server( method=_SOCKS5_AUTH_USER_PASS, auth_status=_SOCKS5_AUTH_FAILURE_STATUS ) with self.assertRaises(_Socks5Error): - self._connect(server, ("localhost", 27017), "user", "password") + self._connect(server, username="user", password="password") server.close() - self.assertEqual( - server.auth, - bytes((_SOCKS5_AUTH_VERSION, len(b"user"))) - + b"user" - + bytes((len(b"password"),)) - + b"password", - ) - def test_connect_request_address_types(self) -> None: - cases = ( + def test_connect_request_and_bound_replies(self) -> None: + for host, expected in ( ("192.0.2.1", bytes((_SOCKS5_ATYP_IPV4,)) + socket.inet_aton("192.0.2.1")), ( "2001:db8::1", @@ -229,13 +165,11 @@ def test_connect_request_address_types(self) -> None: ( "münich.example", bytes((_SOCKS5_ATYP_DOMAIN, len("xn--mnich-kva.example"))) - + "xn--mnich-kva.example".encode("ascii"), + + b"xn--mnich-kva.example", ), - ) - for host, expected in cases: + ): server = self._create_server() - sock = self._connect(server, (host, 27017)) - sock.close() + self._connect(server, (host, 27017)).close() server.close() self.assertEqual( server.request, @@ -244,45 +178,36 @@ def test_connect_request_address_types(self) -> None: + (27017).to_bytes(2, "big"), ) - def test_drains_bound_address_and_preserves_early_payload(self) -> None: for atyp in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): server = self._create_server(bound_atyp=atyp, payload=b"mongo") - sock = self._connect(server, ("localhost", 27017)) - if _IS_SYNC: - payload = sock.recv(5) - else: - sock.setblocking(False) - payload = asyncio.get_running_loop().sock_recv(sock, 5) + sock = self._connect(server) + self.assertEqual(sock.recv(5), b"mongo") sock.close() server.close() - self.assertEqual(payload, b"mongo") - def test_rejects_socks_reply_codes(self) -> None: for reply in range(1, 9): server = self._create_server(reply=reply) with self.subTest(reply=reply), self.assertRaises(_Socks5Error): - self._connect(server, ("localhost", 27017)) + self._connect(server) server.close() - def test_proxy_options_reach_server_and_monitor_pools(self) -> None: - from pymongo.synchronous.mongo_client import MongoClient - from pymongo.synchronous.topology import Topology + def test_tcp_socket_without_cloexec_falls_back(self) -> None: + from pymongo import pool_shared - client = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) - try: - topology = Topology(client._topology_settings) - server_pool = topology._create_pool_for_server(("localhost", 27017)) - monitor_pool = topology._create_pool_for_monitor(("localhost", 27017)) - expected = _ProxyOptions("127.0.0.1") - self.assertEqual(server_pool.opts._proxy_options, expected) - self.assertEqual(monitor_pool.opts._proxy_options, expected) - finally: - client.close() + sock = socket.socket() + self.addCleanup(sock.close) + with mock.patch.object( + pool_shared.socket, "socket", side_effect=[OSError("unsupported"), sock] + ): + self.assertIs(pool_shared._new_tcp_socket(socket.AF_INET, socket.SOCK_STREAM, 0), sock) - def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(self) -> None: + def test_client_proxy_options_and_repr(self) -> None: from pymongo.synchronous.mongo_client import MongoClient + from pymongo.synchronous.topology import Topology - client = MongoClient( + c1 = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) + c2 = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) + c3 = MongoClient( "mongodb://localhost/?proxyHost=localhost&proxyPort=1234" "&proxyUsername=uri-user&proxyPassword=uri-password", proxyHost="[::1]", @@ -291,158 +216,81 @@ def test_kwargs_override_uri_proxy_options_and_empty_values_are_ignored(self) -> proxyPassword="", connect=False, ) - try: - self.assertEqual(client.options.pool_options._proxy_options, _ProxyOptions("::1", 5678)) - finally: - client.close() - - def test_client_equality_includes_proxy_options(self) -> None: - from pymongo.synchronous.mongo_client import MongoClient + c4 = MongoClient( + "mongodb://user:pass@localhost/?proxyHost=127.0.0.1" + "&proxyUsername=proxyuser&proxyPassword=hunter2", + connect=False, + ) + c5 = MongoClient( + "mongodb://localhost/?proxyPort=1081", proxyHost="proxy.example", connect=False + ) + c6 = MongoClient( + "mongodb://localhost/?proxyHost=proxy.example&proxyUsername=u", + proxyPassword="p", + connect=False, + ) + for client in (c1, c2, c3, c4, c5, c6): + self.addCleanup(client.close) - clients = [ - MongoClient("mongodb://localhost/?proxyHost=proxy.example", connect=False), - MongoClient("mongodb://localhost/?proxyHost=proxy.example", connect=False), - MongoClient("mongodb://localhost/?proxyHost=other.example", connect=False), - ] - try: - self.assertEqual(clients[0], clients[1]) - self.assertNotEqual(clients[0], clients[2]) - finally: - for client in clients: - client.close() - - def test_bad_versions_and_unexpected_eof_fail(self) -> None: - cases = ( - (self._create_server(method_version=_SOCKS4_VERSION), ("localhost", 27017), None), - (self._create_server(reply_version=_SOCKS4_VERSION), ("localhost", 27017), None), - (self._create_server(eof_stage="greeting"), ("localhost", 27017), None), - ( - self._create_server(method=_SOCKS5_AUTH_USER_PASS, eof_stage="auth"), - ("localhost", 27017), - ("user", "password"), - ), - (self._create_server(eof_stage="connect"), ("localhost", 27017), None), + topology = Topology(c1._topology_settings) + expected = _ProxyOptions("127.0.0.1") + self.assertEqual( + topology._create_pool_for_server(("localhost", 27017)).opts._proxy_options, expected + ) + self.assertEqual( + topology._create_pool_for_monitor(("localhost", 27017)).opts._proxy_options, expected + ) + self.assertEqual(c1, c2) + self.assertNotEqual(c1, c3) + self.assertEqual(c3.options.pool_options._proxy_options, _ProxyOptions("::1", 5678)) + self.assertEqual( + c5.options.pool_options._proxy_options, _ProxyOptions("proxy.example", 1081) + ) + self.assertEqual( + c6.options.pool_options._proxy_options, _ProxyOptions("proxy.example", 1080, "u", "p") ) - for server, address, credentials in cases: - with self.subTest(stage=server.eof_stage, version=server.method_version): - with self.assertRaises(_Socks5Error): - self._connect(server, address, *(credentials or (None, None))) + for secret in ("hunter2", "proxyuser", "pass"): + self.assertNotIn(secret, repr(c4)) + self.assertNotIn("hunter2", repr(c4.options.pool_options._proxy_options)) + + def test_bad_versions_eof_timeout_and_tls_sni(self) -> None: + for server, creds in ( + (self._create_server(method_version=_SOCKS4_VERSION), (None, None)), + (self._create_server(reply_version=_SOCKS4_VERSION), (None, None)), + (self._create_server(eof_stage="greeting"), (None, None)), + (self._create_server(method=_SOCKS5_AUTH_USER_PASS, eof_stage="auth"), ("u", "p")), + (self._create_server(eof_stage="connect"), (None, None)), + ): + with self.assertRaises(_Socks5Error): + self._connect(server, username=creds[0], password=creds[1]) server.close() - def test_timeout_covers_the_entire_handshake(self) -> None: server = self._create_server(stall=True) with self.assertRaisesRegex(socket.timeout, "SOCKS5 proxy timed out"): - self._connect(server, ("localhost", 27017), timeout=0.1) + self._connect(server, timeout=0.1) server.close() - @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") - def test_cancellation_closes_proxy_socket(self) -> None: - server = self._create_server(stall=True) - task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) - wait_until(server.started.is_set, "receive the SOCKS5 greeting", timeout=2) - task.cancel() - with self.assertRaises(asyncio.CancelledError): - task - - @unittest.skipIf(_IS_SYNC, "Async cancellation only applies to async connections") - def test_cancellation_when_proxy_send_completes_closes_socket(self) -> None: - server = self._create_server(stall=True) - loop = asyncio.get_running_loop() - original_sendall = loop.sock_sendall - sockets: list[socket.socket] = [] - - def cancel_after_send(sock: socket.socket, data: bytes) -> None: - sockets.append(sock) - original_sendall(sock, data) - task.cancel() - - with mock.patch.object(loop, "sock_sendall", new=cancel_after_send): - task = asyncio.create_task(cast(Any, self._connect(server, ("localhost", 27017)))) - with self.assertRaises(asyncio.CancelledError): - task - self.assertEqual(len(sockets), 1) - self.assertEqual(sockets[0].fileno(), -1) - - def test_tls_sni_targets_destination_not_proxy(self) -> None: sni_names: list[Optional[str]] = [] tls_server = _TLSServer(sni_names) self.addCleanup(tls_server.close) proxy = self._create_server(target=tls_server.address) - client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - client_context.check_hostname = False - client_context.verify_mode = ssl.CERT_NONE + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE sock = _configured_socket( ("mongodb.example", tls_server.address[1]), PoolOptions( connect_timeout=2, - ssl_context=cast(Any, client_context), + ssl_context=cast(Any, ctx), proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), ), ) - self.addCleanup(sock.close) sock.close() self.assertEqual(sni_names, ["mongodb.example"]) - def test_proxy_password_is_redacted_in_repr(self) -> None: - from pymongo.synchronous.mongo_client import MongoClient - - for client in ( - MongoClient( - "mongodb://user:dbsecret@localhost/?proxyHost=127.0.0.1" - "&proxyUsername=proxyuser&proxyPassword=hunter2", - connect=False, - ), - MongoClient( - "mongodb://localhost/", - username="user", - password="dbsecret", - proxyHost="127.0.0.1", - proxyUsername="proxyuser", - proxyPassword="hunter2", - connect=False, - ), - ): - try: - client_repr = repr(client) - proxy_repr = repr(client.options.pool_options._proxy_options) - self.assertNotIn("hunter2", client_repr) - self.assertNotIn("proxyuser", client_repr) - self.assertNotIn("dbsecret", client_repr) - self.assertNotIn("hunter2", proxy_repr) - self.assertIn("127.0.0.1", proxy_repr) - finally: - client.close() - - def test_split_proxy_options_across_uri_and_kwargs(self) -> None: - from pymongo.synchronous.mongo_client import MongoClient - - client1 = MongoClient( - "mongodb://localhost/?proxyPort=1081", - proxyHost="proxy.example", - connect=False, - ) - client2 = MongoClient( - "mongodb://localhost/?proxyHost=proxy.example&proxyUsername=u", - proxyPassword="p", - connect=False, - ) - try: - self.assertEqual( - client1.options.pool_options._proxy_options, - _ProxyOptions("proxy.example", 1081), - ) - self.assertEqual( - client2.options.pool_options._proxy_options, - _ProxyOptions("proxy.example", 1080, "u", "p"), - ) - finally: - client1.close() - client2.close() - def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: from pymongo.errors import AutoReconnect, NetworkTimeout from pymongo.pool_shared import _raise_connection_failure - from pymongo.socks5_shared import _Socks5Timeout from pymongo.synchronous.pool import Pool pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) @@ -452,150 +300,80 @@ def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: (_Socks5Timeout("SOCKS5 proxy timed out"), False), (OSError("boom"), True), ): - with self.subTest(cause=type(cause).__name__): - with self.assertRaises((AutoReconnect, NetworkTimeout)) as raised: - _raise_connection_failure(("localhost", 27017), cause) - wrapped = raised.exception - pool._handle_connection_error(wrapped) - self.assertEqual(wrapped.has_error_label("SystemOverloadedError"), overloaded) - self.assertEqual(wrapped.has_error_label("RetryableError"), overloaded) - - def test_multi_address_proxy_fallback_on_connect_error(self) -> None: - dead_addr = ("127.0.0.1", 1) - target = ( - "pymongo.socks5_shared.socket.getaddrinfo" - if _IS_SYNC - else "pymongo.socks5_shared._getaddrinfo" - ) - for error_type in (ConnectionRefusedError, socket.timeout): - server = self._create_server() - fake_results = [ - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr), - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), - ] - - def async_fake_getaddrinfo(*_args: Any, **_kwargs: Any) -> Any: - return fake_results - - replacement: Any = ( - (lambda *_a, **_kw: fake_results) if _IS_SYNC else async_fake_getaddrinfo - ) - with ( - self.subTest(error_type=error_type.__name__), - mock.patch(target, new=replacement), - _fail_connection(dead_addr, error_type), - ): - sock = self._connect(server, ("localhost", 27017)) - sock.close() + with self.assertRaises((AutoReconnect, NetworkTimeout)) as raised: + _raise_connection_failure(("localhost", 27017), cause) + pool._handle_connection_error(raised.exception) + self.assertEqual(raised.exception.has_error_label("SystemOverloadedError"), overloaded) + self.assertEqual(raised.exception.has_error_label("RetryableError"), overloaded) - def test_proxy_connect_timeout_does_not_retry_after_deadline(self) -> None: + def test_sync_proxy_error_and_fallback_paths(self) -> None: import pymongo.synchronous.socks5_support as socks5_module - from pymongo.socks5_shared import _Socks5Timeout - addresses = [ - (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)) - for port in (1, 2) - ] - with ( - mock.patch.object(socks5_module, "_proxy_deadline", return_value=0), - mock.patch.object(socks5_module, "_proxy_getaddrinfo", return_value=addresses), - mock.patch.object( - socks5_module, "_proxy_connect", side_effect=socket.timeout("timed out") - ) as connect, - self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out") as raised, + options = PoolOptions(connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080)) + target = "pymongo.socks5_shared.socket.getaddrinfo" + for err, exc_type in ( + (socket.gaierror("failed"), _Socks5Error), + (socket.timeout("timed out"), _Socks5Timeout), ): - _create_connection( - ("mongodb.example", 27017), - PoolOptions(proxy_options=_ProxyOptions("proxy.example")), - ) - connect.assert_called_once() - self.assertEqual(connect.call_args.args[0].fileno(), -1) - self.assertIsInstance(raised.exception.__cause__, socket.timeout) - - def test_proxy_connection_error_paths(self) -> None: - proxy_opts = PoolOptions( - connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080) - ) - target = ( - "pymongo.socks5_shared.socket.getaddrinfo" - if _IS_SYNC - else "pymongo.socks5_shared._getaddrinfo" - ) - - def raise_gai(*_a: Any, **_kw: Any) -> Any: - raise socket.gaierror("resolution failed") + with mock.patch(target, side_effect=err), self.assertRaises(exc_type): + _create_connection(("localhost", 27017), options) with ( - mock.patch(target, side_effect=raise_gai), - self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), + mock.patch(target, return_value=[]), + self.assertRaisesRegex(_Socks5Error, "no addresses"), ): - _create_connection(("localhost", 27017), proxy_opts) + _create_connection(("localhost", 27017), options) - with ( - mock.patch(target, side_effect=socket.timeout("timed out")), - self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), + dead = (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", 1)) + for err, exc_type in ( + (ConnectionRefusedError("refused"), _Socks5Error), + (socket.timeout(), _Socks5Timeout), ): - _create_connection(("localhost", 27017), proxy_opts) - - def empty_async(*_a: Any, **_kw: Any) -> Any: - return [] - - replacement_empty: Any = (lambda *_a, **_kw: []) if _IS_SYNC else empty_async - with ( - mock.patch(target, new=replacement_empty), - self.assertRaisesRegex(_Socks5Error, "returned no addresses"), - ): - _create_connection(("localhost", 27017), proxy_opts) - - dead_addr = ("127.0.0.1", 1) - dead_results = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead_addr)] - - def dead_async(*_a: Any, **_kw: Any) -> Any: - return dead_results + with ( + mock.patch(target, return_value=[dead]), + mock.patch.object(socks5_module, "_proxy_connect", side_effect=err), + self.assertRaises(exc_type), + ): + _create_connection(("localhost", 27017), options) - replacement_dead: Any = (lambda *_a, **_kw: dead_results) if _IS_SYNC else dead_async - with ( - mock.patch(target, new=replacement_dead), - _fail_connection(dead_addr), - self.assertRaisesRegex(_Socks5Error, "proxy connection failed"), - ): - _create_connection(("localhost", 27017), proxy_opts) + server = self._create_server() + live = (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address) + orig_connect = socks5_module._proxy_connect - with ( - mock.patch(target, new=replacement_dead), - _fail_connection(dead_addr, socket.timeout), - self.assertRaisesRegex(_Socks5Timeout, "SOCKS5 proxy timed out"), - ): - _create_connection(("localhost", 27017), proxy_opts) + def fail_first(sock: socket.socket, addr: Any, deadline: Optional[float]) -> None: + if addr == ("127.0.0.1", 1): + raise ConnectionRefusedError("refused") + orig_connect(sock, addr, deadline) - server = self._create_server() - handshake_options = PoolOptions( - connect_timeout=1, - proxy_options=_ProxyOptions(server.address[0], server.address[1]), - ) - sendall_target = ( - "pymongo.synchronous.socks5_support._socks5_sendall" - if _IS_SYNC - else "pymongo.socks5_support._socks5_sendall" - ) with ( - mock.patch(sendall_target, side_effect=OSError("handshake failed")), - self.assertRaisesRegex(_Socks5Error, "SOCKS5 handshake failed"), + mock.patch(target, return_value=[dead, live]), + mock.patch.object(socks5_module, "_proxy_connect", side_effect=fail_first), ): - _create_connection(("localhost", 27017), handshake_options) + self._connect(server).close() server.close() - server = self._create_server() - handshake_options = PoolOptions( - connect_timeout=1, - proxy_options=_ProxyOptions(server.address[0], server.address[1]), - ) with ( - mock.patch(sendall_target, side_effect=KeyboardInterrupt), - self.assertRaises(KeyboardInterrupt), + mock.patch.object(socks5_module, "_proxy_deadline", return_value=0), + mock.patch.object(socks5_module, "_proxy_getaddrinfo", return_value=[dead, dead]), + mock.patch.object( + socks5_module, "_proxy_connect", side_effect=socket.timeout("timed out") + ) as connect, + self.assertRaises(_Socks5Timeout), ): - _create_connection(("localhost", 27017), handshake_options) - server.close() + _create_connection(("mongodb.example", 27017), options) + connect.assert_called_once() + + for error in (OSError("handshake failed"), KeyboardInterrupt): + server = self._create_server() + opts = PoolOptions(connect_timeout=1, proxy_options=_ProxyOptions(*server.address)) + with ( + mock.patch.object(socks5_module, "_socks5_sendall", side_effect=error), + self.assertRaises( + _Socks5Error if isinstance(error, OSError) else KeyboardInterrupt + ), + ): + _create_connection(("localhost", 27017), opts) + server.close() class _CommandListener(monitoring.CommandListener): @@ -612,138 +390,557 @@ def failed(self, event: monitoring.CommandFailedEvent) -> None: self.connection_ids.append(event.connection_id) +def _start_socks5_proxies(nodes: list[tuple[str, int]]) -> list[subprocess.Popen[bytes]]: + tools = Path(os.environ.get("DRIVERS_TOOLS", "drivers-evergreen-tools")) + script = tools / ".evergreen" / "socks5srv.py" + host, port = nodes[0] + seed = f"[{host}]:{port}" if ":" in host else f"{host}:{port}" + mapping = f"localhost:12345 to {seed}" + processes = [] + try: + for proxy_port, auth in ((1080, True), (1081, False)): + args = [sys.executable, str(script), "--port", str(proxy_port), "--map", mapping] + if auth: + args.extend(("--auth", "username:p4ssw0rd")) + process = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + processes.append(process) + deadline = time.monotonic() + 10 + while True: + if process.poll() is not None: + raise RuntimeError(f"SOCKS5 proxy on port {proxy_port} exited") + try: + with socket.create_connection(("127.0.0.1", proxy_port), timeout=0.1): + break + except OSError: + if time.monotonic() >= deadline: + raise RuntimeError(f"SOCKS5 proxy on port {proxy_port} did not start") + time.sleep(0.1) + return processes + except BaseException: + _stop_socks5_proxies(processes) + raise + + +def _stop_socks5_proxies(processes: list[subprocess.Popen[bytes]]) -> None: + for process in processes: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +async def _maybe_await(value: Any) -> Any: + return await value if hasattr(value, "__await__") else value + + +def _prose_hosts(nodes: list[tuple[str, int]], mapped: bool) -> str: + if mapped: + return "localhost:12345" + return ",".join( + f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" + for host, port in nodes + ) + + +async def _run_prose_case( + test: Any, + async_api: bool, + hosts: str, + port: int, + credentials: Optional[tuple[str, str]], + succeeds: bool, + use_uri: bool, + listener: Optional[_CommandListener] = None, +) -> None: + from pymongo.errors import ConnectionFailure, OperationFailure + + if async_api: + from pymongo.asynchronous.mongo_client import AsyncMongoClient as Client + else: + from pymongo.synchronous.mongo_client import MongoClient as Client + proxy = {"proxyHost": "localhost", "proxyPort": port} + if credentials: + proxy.update(proxyUsername=credentials[0], proxyPassword=credentials[1]) + context = async_client_context if async_api else client_context + options = dict(TLS_OPTIONS) if context.tls else {} + options.update(test.credentials) + options["serverSelectionTimeoutMS"] = 2500 + if hosts == "localhost:12345": + options["directConnection"] = True + if listener: + options["event_listeners"] = [listener] + if use_uri: + query = "&".join(f"{key}={quote_plus(str(value))}" for key, value in proxy.items()) + hosts = f"mongodb://{hosts}/?{query}" + else: + options.update(proxy) + client = Client(hosts, **options) + try: + try: + await _maybe_await(client.admin.command(HelloCompat.CMD)) + except ConnectionFailure: + if succeeds: + raise + return + if not succeeds: + raise AssertionError("Expected the SOCKS5 connection to fail") + if listener: + with pytest.raises(OperationFailure): + await _maybe_await(client.admin.command("nonexistent_socks5_test_command")) + finally: + await _maybe_await(client.close()) + + +async def _test_prose_table(test: Any, async_api: bool) -> None: + cases = ( + _ProseCase(True, 1080, None, False), + _ProseCase(True, 1081, None, True), + _ProseCase(False, 1080, None, False), + _ProseCase(False, 1081, None, True), + _ProseCase(True, 1080, ("nonexistentuser", "badauth"), False), + _ProseCase(True, 1081, ("nonexistentuser", "badauth"), True), + _ProseCase(False, 1081, ("nonexistentuser", "badauth"), True), + _ProseCase(True, 1080, ("username", "p4ssw0rd"), True), + _ProseCase(False, 1080, ("username", "p4ssw0rd"), True), + ) + for case in cases: + for use_uri in (True, False): + await _run_prose_case( + test, + async_api, + _prose_hosts(test.nodes, case.mapped), + case.proxy_port, + case.credentials, + case.succeeds, + use_uri, + ) + + +async def _test_command_monitoring(test: Any, async_api: bool) -> None: + proxy_addresses = {(host, port) for host in ("localhost", "127.0.0.1") for port in (1080, 1081)} + for mapped in (True, False): + listener = _CommandListener() + await _run_prose_case( + test, async_api, _prose_hosts(test.nodes, mapped), 1081, None, True, False, listener + ) + assert len(listener.connection_ids) >= 4 + for address in listener.connection_ids: + assert address not in proxy_addresses + assert address == (("localhost", 12345) if mapped else address) + if not mapped: + assert address in test.nodes + + class TestSocks5Prose(IntegrationTest): @client_context.require_replica_set def setUp(self) -> None: - if not os.environ.get("SOCKS5_PROXIES_STARTED"): - self.skipTest("SOCKS5 proxies must be started by the Evergreen SOCKS5 tasks.") super().setUp() + if not os.environ.get("TEST_SOCKS5"): + self.skipTest("SOCKS5 proxies are started only by the SOCKS5 task") self.nodes = sorted(client_context.nodes) - self.auth_port = 1080 - self.noauth_port = 1081 - - def _hosts(self, mapped: bool) -> str: - if mapped: - return "localhost:12345" - return ",".join( - f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" - for host, port in self.nodes - ) + self.addCleanup(_stop_socks5_proxies, _start_socks5_proxies(self.nodes)) - def _run_case( - self, - hosts: str, - port: int, - credentials: Optional[tuple[str, str]], - succeeds: bool, - use_uri: bool, - listener: Optional[_CommandListener] = None, - ) -> None: - from pymongo.synchronous.mongo_client import MongoClient + def test_connection_string_prose_table(self) -> None: + asyncio.run(_test_prose_table(self, False)) - proxy_options = {"proxyHost": "localhost", "proxyPort": port} - if credentials: - proxy_options["proxyUsername"], proxy_options["proxyPassword"] = credentials - tls_options = dict(TLS_OPTIONS) if client_context.tls else {} - options = dict(tls_options) - options.update(self.credentials) - options["serverSelectionTimeoutMS"] = 2500 - if hosts == "localhost:12345": - options["directConnection"] = True - if listener is not None: - options["event_listeners"] = [listener] - if use_uri: - uri_options = "&".join( - f"{key}={quote_plus(str(value))}" for key, value in proxy_options.items() - ) - hosts = f"mongodb://{hosts}/?{uri_options}" - else: - options.update(proxy_options) - client = MongoClient(hosts, **options) + def test_command_monitoring_does_not_reference_proxy(self) -> None: + asyncio.run(_test_command_monitoring(self, False)) + + +class _TLSServer: + def __init__(self, sni_names: list[Optional[str]]) -> None: + self.sni_names = sni_names + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.listener.settimeout(3) + self.address = self.listener.getsockname() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + def _serve(self) -> None: try: - if succeeds: - client.admin.command(HelloCompat.CMD) - if listener is not None: - from pymongo.errors import OperationFailure + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(f"{CERT_PATH}/server.pem") + context.set_servername_callback(lambda _sock, name, _ctx: self.sni_names.append(name)) + with self.listener.accept()[0] as raw: + with context.wrap_socket(raw, server_side=True) as conn: + conn.recv(1) + except OSError: + pass + finally: + self.listener.close() - with self.assertRaises(OperationFailure): - client.admin.command("nonexistent_socks5_test_command") - else: - from pymongo.errors import ConnectionFailure + def close(self) -> None: + self.listener.close() + self.thread.join(timeout=3) - with self.assertRaises(ConnectionFailure): - client.admin.command(HelloCompat.CMD) + +class _Socks5Server: + def __init__( + self, + method: int = 0x00, + auth_status: int = 0x00, + reply: int = 0x00, + bound_atyp: int = 0x01, + payload: bytes = b"", + method_version: int = 0x05, + reply_version: int = 0x05, + eof_stage: Optional[str] = None, + stall: bool = False, + target: Optional[tuple[str, int]] = None, + ) -> None: + self.method = method + self.auth_status = auth_status + self.reply = reply + self.bound_atyp = bound_atyp + self.payload = payload + self.method_version = method_version + self.reply_version = reply_version + self.eof_stage = eof_stage + self.stall = stall + self.target = target + self.started = threading.Event() + self.greeting = b"" + self.auth = b"" + self.request = b"" + self.error: Optional[BaseException] = None + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.listener.settimeout(3) + self.address = self.listener.getsockname() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + @staticmethod + def _read(sock: socket.socket, size: int) -> bytes: + data = bytearray() + while len(data) < size: + part = sock.recv(size - len(data)) + if not part: + raise EOFError("client closed connection") + data.extend(part) + return bytes(data) + + def _serve(self) -> None: + upstream: Optional[socket.socket] = None + try: + with self.listener.accept()[0] as conn: + conn.settimeout(3) + version, nmethods = self._read(conn, 2) + self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) + self.started.set() + if self.stall: + conn.recv(1) + return + if self.eof_stage == "greeting": + conn.sendall(bytes((0x05,))) + return + conn.sendall(bytes((self.method_version, self.method))) + if self.method == 0x02: + auth_version, username_length = self._read(conn, 2) + username = self._read(conn, username_length) + password_length = self._read(conn, 1)[0] + password = self._read(conn, password_length) + self.auth = ( + bytes((auth_version, username_length)) + + username + + bytes((password_length,)) + + password + ) + if self.eof_stage == "auth": + conn.sendall(bytes((0x01,))) + return + conn.sendall(bytes((0x01, self.auth_status))) + if self.auth_status: + return + version, command, reserved, atyp = self._read(conn, 4) + if atyp == 0x01: + addr = self._read(conn, 4) + elif atyp == 0x04: + addr = self._read(conn, 16) + else: + length = self._read(conn, 1) + addr = length + self._read(conn, length[0]) + self.request = ( + bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) + ) + if self.eof_stage == "connect": + conn.sendall(bytes((0x05, 0x00))) + return + if self.target is not None: + upstream = socket.create_connection(self.target, timeout=3) + if self.bound_atyp == 0x01: + bound = bytes((0x01,)) + socket.inet_aton("127.0.0.1") + bytes(2) + elif self.bound_atyp == 0x04: + bound = bytes((0x04,)) + socket.inet_pton(socket.AF_INET6, "::1") + bytes(2) + else: + host = b"x" + bound = bytes((0x03, len(host))) + host + bytes(2) + conn.sendall(bytes((self.reply_version, self.reply, 0x00)) + bound + self.payload) + if self.payload: + return + if upstream is not None: + conn.setblocking(False) + upstream.setblocking(False) + while True: + readable, _, _ = select.select([conn, upstream], [], [], 5) + if not readable: + return + for source in readable: + data = source.recv(65536) + if not data: + return + destination = upstream if source is conn else conn + destination.sendall(data) + else: + while conn.recv(1024): + pass + except BaseException as exc: + self.error = exc finally: - client.close() + if upstream is not None: + upstream.close() + self.listener.close() - def test_connection_string_prose_table(self) -> None: - cases = ( - _ProseCase(mapped=True, proxy_port=self.auth_port, credentials=None, succeeds=False), - _ProseCase(mapped=True, proxy_port=self.noauth_port, credentials=None, succeeds=True), - _ProseCase(mapped=False, proxy_port=self.auth_port, credentials=None, succeeds=False), - _ProseCase(mapped=False, proxy_port=self.noauth_port, credentials=None, succeeds=True), - _ProseCase( - mapped=True, - proxy_port=self.auth_port, - credentials=("nonexistentuser", "badauth"), - succeeds=False, - ), - _ProseCase( - mapped=True, - proxy_port=self.noauth_port, - credentials=("nonexistentuser", "badauth"), - succeeds=True, + def close(self) -> None: + self.listener.close() + self.thread.join(timeout=3) + + +@pytest.mark.default_async +class TestAsyncSocks5(AsyncPyMongoTestCase): + """Exercise the asynchronous proxy paths without duplicating sync scenarios.""" + + def _create_server(self, **kwargs: Any) -> _Socks5Server: + server = _Socks5Server(**kwargs) + self.addAsyncCleanup(server.close) + return server + + async def _connect( + self, + server: _Socks5Server, + address: tuple[str, int] = ("localhost", 27017), + username: Optional[str] = None, + password: Optional[str] = None, + timeout: float = 2, + ) -> socket.socket: + from pymongo.pool_shared import _async_create_connection + + sock = await _async_create_connection( + address, + PoolOptions( + connect_timeout=timeout, + proxy_options=_ProxyOptions( + server.address[0], server.address[1], username, password + ), ), - _ProseCase( - mapped=False, - proxy_port=self.noauth_port, - credentials=("nonexistentuser", "badauth"), - succeeds=True, + ) + self.addAsyncCleanup(sock.close) + return sock + + async def test_protocol_and_connection_errors(self) -> None: + for host, atyp in (("127.0.0.1", 1), ("2001:db8::1", 4), ("münich.example", 3)): + server = self._create_server() + sock = await self._connect(server, (host, 27017)) + sock.close() + server.close() + self.assertEqual(server.request[3], atyp) + + server = self._create_server(method=2) + sock = await self._connect(server, username="üser", password="påss") + sock.close() + server.close() + self.assertIn("üser".encode(), server.auth) + + for method in (1, 0xFF): + server = self._create_server(method=method) + with self.assertRaises(_Socks5Error): + await self._connect(server) + server.close() + server = self._create_server(method=2, auth_status=1) + with self.assertRaises(_Socks5Error): + await self._connect(server, username="u", password="p") + server.close() + for reply in range(1, 9): + server = self._create_server(reply=reply) + with self.assertRaises(_Socks5Error): + await self._connect(server) + server.close() + server = self._create_server(eof_stage="greeting") + with self.assertRaisesRegex(_Socks5Error, "unexpected EOF"): + await self._connect(server) + server.close() + + async def test_async_client_options_and_configured_socket(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + from pymongo.asynchronous.uri_parser import parse_uri + from pymongo.pool_shared import _async_configured_socket + + self.assertEqual( + (await parse_uri("mongodb://localhost/?proxyHost=localhost"))["options"]["proxyHost"], + "localhost", + ) + + server = self._create_server() + client = AsyncMongoClient( + "mongodb://localhost/?proxyHost=localhost&proxyPort=1234", + proxyHost=server.address[0], + proxyPort=server.address[1], + connect=False, + ) + self.assertEqual( + client.options.pool_options._proxy_options, + _ProxyOptions(server.address[0], server.address[1]), + ) + await client.close() + + tls = _TLSServer([]) + self.addAsyncCleanup(tls.close) + proxy = self._create_server(target=tls.address) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + sock = await _async_configured_socket( + ("mongodb.example", tls.address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, context), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), ), - _ProseCase( - mapped=True, - proxy_port=self.auth_port, - credentials=("username", "p4ssw0rd"), - succeeds=True, + ) + sock.close() + self.assertEqual(tls.sni_names, ["mongodb.example"]) + + async def test_async_proxy_error_paths(self) -> None: + from pymongo.asynchronous import socks5_support + from pymongo.pool_shared import _async_create_connection + + options = PoolOptions(proxy_options=_ProxyOptions("proxy.invalid", 1080)) + for error, expected in ( + (TimeoutError("timed out"), _Socks5Timeout), + (OSError("resolution failed"), _Socks5Error), + ): + with ( + mock.patch.object(socks5_support, "_async_proxy_getaddrinfo", side_effect=error), + self.assertRaises(expected), + ): + await _async_create_connection(("localhost", 27017), options) + + addresses = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", 1))] + + async def fake_addresses(*_args: Any, **_kwargs: Any) -> Any: + return addresses + + for error, expected in ( + (TimeoutError("timed out"), _Socks5Timeout), + (OSError("refused"), _Socks5Error), + ): + with ( + mock.patch.object(socks5_support, "_async_proxy_getaddrinfo", new=fake_addresses), + mock.patch.object(socks5_support, "_async_proxy_connect", side_effect=error), + self.assertRaises(expected), + ): + await _async_create_connection(("localhost", 27017), options) + with ( + mock.patch.object(socks5_support, "_async_proxy_getaddrinfo", new=fake_addresses), + mock.patch.object(socks5_support, "_async_proxy_connect", side_effect=socket.timeout()), + mock.patch.object( + socks5_support, "_proxy_remaining", side_effect=_Socks5Timeout("expired") ), - _ProseCase( - mapped=False, - proxy_port=self.auth_port, - credentials=("username", "p4ssw0rd"), - succeeds=True, + self.assertRaises(_Socks5Timeout), + ): + await _async_create_connection(("localhost", 27017), options) + + async def no_addresses(*_args: Any, **_kwargs: Any) -> Any: + return [] + + with ( + mock.patch.object(socks5_support, "_async_proxy_getaddrinfo", new=no_addresses), + self.assertRaisesRegex(_Socks5Error, "returned no addresses"), + ): + await _async_create_connection(("localhost", 27017), options) + + for error in (OSError("handshake failed"), KeyboardInterrupt): + server = self._create_server() + options = PoolOptions(proxy_options=_ProxyOptions(server.address[0], server.address[1])) + with ( + mock.patch.object(socks5_support, "_async_socks5_sendall", side_effect=error), + self.assertRaisesRegex( + _Socks5Error if isinstance(error, OSError) else KeyboardInterrupt, + "handshake failed" if isinstance(error, OSError) else None, + ), + ): + await _async_create_connection(("localhost", 27017), options) + server.close() + + async def test_async_timeout_cancellation_and_proxy_fallback(self) -> None: + from pymongo.pool_shared import _async_create_connection + + server = self._create_server(stall=True) + with self.assertRaisesRegex(socket.timeout, "SOCKS5 proxy timed out"): + await self._connect(server, timeout=0.1) + server.close() + + server = self._create_server(stall=True) + task = asyncio.create_task(self._connect(server)) + await async_wait_until(server.started.is_set, "receive SOCKS5 greeting", timeout=2) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + server.close() + + dead = ("127.0.0.1", 1) + server = self._create_server() + fake_results = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), + ] + loop = asyncio.get_running_loop() + orig_connect = loop.sock_connect + + async def fail_first(sock: socket.socket, target: tuple[str, int]) -> None: + if target == dead: + raise ConnectionRefusedError("connection failed") + await orig_connect(sock, target) + + with ( + mock.patch( + "pymongo.socks5_shared._getaddrinfo", new=mock.AsyncMock(return_value=fake_results) ), - ) - for case in cases: - for use_uri in (True, False): - with self.subTest( - mapped=case.mapped, - proxy_port=case.proxy_port, - credentials=case.credentials, - use_uri=use_uri, - ): - self._run_case( - hosts=self._hosts(case.mapped), - port=case.proxy_port, - credentials=case.credentials, - succeeds=case.succeeds, - use_uri=use_uri, - ) + mock.patch.object(loop, "sock_connect", new=fail_first), + ): + (await self._connect(server)).close() + server.close() - def test_command_monitoring_does_not_reference_proxy(self) -> None: - proxy_addresses = { - ("localhost", self.auth_port), - ("localhost", self.noauth_port), - ("127.0.0.1", self.auth_port), - ("127.0.0.1", self.noauth_port), - } - for mapped in (True, False): - listener = _CommandListener() - with self.subTest(mapped=mapped): - self._run_case(self._hosts(mapped), self.noauth_port, None, True, False, listener) - self.assertGreaterEqual(len(listener.connection_ids), 4) - for connection_id in listener.connection_ids: - self.assertNotIn(connection_id, proxy_addresses) - if mapped: - self.assertEqual(connection_id, ("localhost", 12345)) - else: - self.assertIn(connection_id, self.nodes) + with ( + mock.patch("pymongo.socks5_shared._getaddrinfo", side_effect=socket.gaierror("failed")), + self.assertRaises(_Socks5Error), + ): + await _async_create_connection( + ("localhost", 27017), + PoolOptions(proxy_options=_ProxyOptions("proxy.invalid", 1080)), + ) + + +@pytest.mark.default_async +class TestSocks5ProseAsync(AsyncIntegrationTest): + @async_client_context.require_replica_set + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + if not os.environ.get("TEST_SOCKS5"): + self.skipTest("SOCKS5 proxies are started only by the SOCKS5 task") + self.nodes = sorted(async_client_context.nodes) + self.addAsyncCleanup(_stop_socks5_proxies, _start_socks5_proxies(self.nodes)) + + async def test_connection_string_prose_table(self) -> None: + await _test_prose_table(self, True) + + async def test_command_monitoring_does_not_reference_proxy(self) -> None: + await _test_command_monitoring(self, True) diff --git a/test/test_uri_parser.py b/test/test_uri_parser.py index c0c4897792..d2c3d4ffe8 100644 --- a/test/test_uri_parser.py +++ b/test/test_uri_parser.py @@ -659,149 +659,70 @@ def test_split_options_duplicate_warning(self): def test_split_options_empty_authsource(self): self.assertRaises(InvalidURI, split_options, "authSource=") - def test_socks5_proxy_options(self): - parsed = parse_uri( - "mongodb://localhost/?proxyHost=%5B%3A%3A1%5D&proxyPort=1234" - "&proxyUsername=user&proxyPassword=pass" + def test_socks5_validation(self): + uri = ( + "mongodb://localhost/?proxyHost=%5B%3A%3A1%5D&proxyPort=invalid&proxyUsername=" + + "é" * 128 + + "&proxyPassword=pass" ) + self.assertEqual(parse_uri(uri, validate=False)["options"]["proxyPort"], "invalid") self.assertEqual( - parsed["options"], - { - "proxyHost": "::1", - "proxyPort": 1234, - "proxyUsername": "user", - "proxyPassword": "pass", - }, - ) - empty_credentials = parse_uri( - "mongodb://localhost/?proxyHost=localhost&proxyUsername=&proxyPassword=" + parse_uri("mongodb://localhost/?proxyUsername=&proxyPassword=", normalize=False)[ + "options" + ], + {"proxyUsername": "", "proxyPassword": ""}, ) - self.assertEqual(empty_credentials["options"], {"proxyHost": "localhost"}) - self.assertEqual( - parse_uri("mongodb://localhost/?proxyHost=localhost")["options"]["proxyHost"], - "localhost", - ) - - def test_socks5_invalid_proxy_options(self): - for query in ( - "proxyHost=", - "proxyHost=%5B%3A%3A1", - "proxyHost=%5B%3A%3A1%5D%3A1080", - "proxyHost=localhost%3A1080", - "proxyHost=local%20host", - "proxyPort=1080", - "proxyUsername=user", - "proxyPassword=pass", - "proxyHost=localhost&proxyUsername=user", - "proxyHost=localhost&proxyPassword=pass", - "proxyHost=localhost&proxyPort=0", - "proxyHost=localhost&proxyPort=-1", - "proxyHost=localhost&proxyPort=65536", - "proxyHost=localhost&proxyPort=abc", - "proxyHost=localhost&proxyUsername=" + ("é" * 128) + "&proxyPassword=pass", - "proxyHost=localhost&proxyUsername=user&proxyPassword=" + ("é" * 128), - ): + for query in ("proxyHost=", "proxyPort=0", "proxyPort=65536", "proxyUsername=user"): with ( - self.subTest(query=query[:60]), + self.subTest(query=query), self.assertRaises((InvalidURI, ValueError, ConfigurationError)), ): parse_uri("mongodb://localhost/?" + query) + with self.assertRaises(InvalidURI): + parse_uri("mongodb://localhost/?proxyHost=localhost&proxyhost=a&proxyhost=b") + with self.assertRaises(ConfigurationError): + parse_uri("mongodb://%2Ftmp%2Fmongodb.sock/?proxyHost=localhost") - for option in ("proxyhost", "proxyport", "proxyusername", "proxypassword"): - uri = "mongodb://localhost/?proxyHost=localhost&" + option + "=a&" + option + "=b" - with self.subTest(option=option), self.assertRaises(InvalidURI): - parse_uri(uri) - - def test_socks5_skips_individual_validation_when_disabled(self): - username = "é" * 128 - parsed = parse_uri( - "mongodb://localhost/?proxyHost=%5B%3A%3A1%5D&proxyPort=invalid" - f"&proxyUsername={username}&proxyPassword=pass", - validate=False, - ) - self.assertEqual( - parsed["options"], - { - "proxyHost": "[::1]", - "proxyPort": "invalid", - "proxyUsername": username, - "proxyPassword": "pass", - }, - ) - self.assertEqual( - parse_uri("mongodb://localhost/?proxyHost=invalid%2Fhost", validate=False)["options"], - {"proxyHost": "invalid/host"}, - ) - - def test_socks5_empty_credentials_without_normalization(self): - for validate in (True, False): - with self.subTest(validate=validate): - parsed = parse_uri( - "mongodb://localhost/?proxyUsername=&proxyPassword=", - validate=validate, - normalize=False, - ) - self.assertEqual(parsed["options"], {"proxyUsername": "", "proxyPassword": ""}) - - def test_socks5_validator_functions(self): - self.assertIsNone(validate_proxy_host("proxyHost", None)) + def test_socks5_validators(self): self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") - with self.assertRaises(ValueError): - validate_proxy_host("proxyHost", "[not-ipv6]") - with self.assertRaises(ValueError): - validate_proxy_host("proxyHost", "bad/host") - + self.assertIsNone(validate_proxy_host("proxyHost", None)) + for host in ("bad/host", "bad host", "[::1", "[invalid]", "localhost:1080"): + with self.subTest(host=host), self.assertRaises(ValueError): + validate_proxy_host("proxyHost", host) self.assertIsNone(validate_proxy_port("proxyPort", None)) self.assertEqual(validate_proxy_port("proxyPort", 1080), 1080) with self.assertRaises(TypeError): validate_proxy_port("proxyPort", True) - - valid = "é" * 127 + "a" - for option in ("proxyUsername", "proxyPassword"): - with self.subTest(option=option): - self.assertIsNone(validate_proxy_credential(option, None)) - self.assertEqual(validate_proxy_credential(option, ""), "") - self.assertEqual(validate_proxy_credential(option, valid), valid) - with self.assertRaisesRegex(ValueError, "no more than 255 UTF-8 bytes"): - validate_proxy_credential(option, "é" * 128) - with self.assertRaises(TypeError): - validate_proxy_credential(option, 1) + for key in ("proxyUsername", "proxyPassword"): + self.assertEqual(validate_proxy_credential(key, ""), "") + self.assertEqual(validate_proxy_credential(key, "é" * 127 + "a"), "é" * 127 + "a") + with self.assertRaises(ValueError): + validate_proxy_credential(key, "é" * 128) + with self.assertRaises(TypeError): + validate_proxy_credential(key, 1) def test_socks5_deadline_helpers(self): self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) self.assertIsNone(_proxy_remaining(None)) with self.assertRaises(_Socks5Timeout): _proxy_remaining(0.0) - with ( - patch("pymongo.socks5_shared._csot.remaining", return_value=0.0), - self.assertRaises(_Socks5Timeout), - ): - _proxy_deadline(PoolOptions(connect_timeout=1)) - with ( - patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1), - self.assertRaises(_Socks5Timeout), - ): - _proxy_socket_timeout(PoolOptions(connect_timeout=1)) - for remaining, expected in ((None, 2), (0.5, 0.5)): - with ( - self.subTest(remaining=remaining), - patch("pymongo.socks5_shared._csot.remaining", return_value=remaining), - patch("pymongo.socks5_shared.time.monotonic", return_value=100), - ): - options = PoolOptions(connect_timeout=2) - self.assertEqual(_proxy_socket_timeout(options), expected) - self.assertEqual(_proxy_deadline(options), 100 + expected) + with patch("pymongo.socks5_shared._csot.remaining", return_value=0.0): + with self.assertRaises(_Socks5Timeout): + _proxy_deadline(PoolOptions(connect_timeout=1)) + with patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1): + with self.assertRaises(_Socks5Timeout): + _proxy_socket_timeout(PoolOptions(connect_timeout=1)) def test_socks5_protocol_validation(self): - with self.assertRaisesRegex(_Socks5Error, "missing bound domain length"): - _socks5_bound_address_size(3) + self.assertEqual(_socks5_bound_address_size(1), 6) + self.assertEqual(_socks5_bound_address_size(4), 18) + self.assertEqual(_socks5_bound_address_size(3, 5), 7) for address, message in ( (("a" * 256, 27017), "invalid domain name length"), ((".münich.example", 27017), "invalid domain name"), - (("a" * 256, None), "destination port is required"), ): with self.subTest(address=address), self.assertRaisesRegex(_Socks5Error, message): - _socks5_request(address) # type: ignore[arg-type] + _socks5_request(address) for reply, message in ( (b"\x05\x00\x01\x01", "invalid reserved byte"), (b"\x05\x00\x00\x09", "invalid address type"), @@ -809,10 +730,6 @@ def test_socks5_protocol_validation(self): with self.subTest(reply=reply), self.assertRaisesRegex(_Socks5Error, message): _socks5_check_connect_reply(reply) - def test_socks5_proxy_rejects_unix_socket(self): - with self.assertRaises(ConfigurationError): - parse_uri("mongodb://%2Ftmp%2Fmongodb.sock/?proxyHost=localhost") - def test_check_options_conflicts(self): self.assertRaises( ConfigurationError, From e9ba031f64e65244441720e1ef64bcb34205bf73 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Sat, 7 Nov 2026 11:16:37 +0000 Subject: [PATCH 16/16] address review feedback --- pymongo/asynchronous/encryption.py | 3 +- pymongo/synchronous/encryption.py | 3 +- test/test_kms_connect.py | 2 +- test/test_socks5.py | 60 ++++++++++++++++++------------ 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 3eee5c779e..4cff2ec34f 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -125,7 +125,7 @@ def __init__( key_vault_coll: AsyncCollection[_DocumentTypeArg], mongocryptd_client: Optional[AsyncMongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, - proxy_options: Optional[_ProxyOptions], + proxy_options: Optional[_ProxyOptions] = None, ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -395,6 +395,7 @@ def _get_internal_client( encrypter._internal_client = internal_client return internal_client + # KMS requests use the encrypted client's SOCKS5 proxy options, not the key vault client's. if opts._key_vault_client is not None: key_vault_client = opts._key_vault_client else: diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 05c727fadd..8713697b36 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -124,7 +124,7 @@ def __init__( key_vault_coll: Collection[_DocumentTypeArg], mongocryptd_client: Optional[MongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, - proxy_options: Optional[_ProxyOptions], + proxy_options: Optional[_ProxyOptions] = None, ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -392,6 +392,7 @@ def _get_internal_client( encrypter._internal_client = internal_client return internal_client + # KMS requests use the encrypted client's SOCKS5 proxy options, not the key vault client's. if opts._key_vault_client is not None: key_vault_client = opts._key_vault_client else: diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index 887ff56888..4b06a1c1d3 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -1149,7 +1149,7 @@ def action(context): return "not-a-socket" opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=api.callback(action)) - io = api.encryption()._EncryptionIO(None, mock.MagicMock(), None, opts, None) + io = api.encryption()._EncryptionIO(None, mock.MagicMock(), None, opts) class StubKmsContext: endpoint = "kms.example.com:443" diff --git a/test/test_socks5.py b/test/test_socks5.py index bdbd41d3f8..4d93046409 100644 --- a/test/test_socks5.py +++ b/test/test_socks5.py @@ -49,9 +49,11 @@ _SOCKS5_ATYP_IPV6, _SOCKS5_AUTH_NO_ACCEPTABLE, _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_SUCCESS, _SOCKS5_AUTH_USER_PASS, _SOCKS5_AUTH_VERSION, _SOCKS5_CMD_CONNECT, + _SOCKS5_REPLY_SUCCESS, _SOCKS5_RESERVED, _SOCKS5_VERSION, _Socks5Error, @@ -583,13 +585,13 @@ def close(self) -> None: class _Socks5Server: def __init__( self, - method: int = 0x00, - auth_status: int = 0x00, - reply: int = 0x00, - bound_atyp: int = 0x01, + method: int = _SOCKS5_AUTH_NO_AUTH, + auth_status: int = _SOCKS5_AUTH_SUCCESS, + reply: int = _SOCKS5_REPLY_SUCCESS, + bound_atyp: int = _SOCKS5_ATYP_IPV4, payload: bytes = b"", - method_version: int = 0x05, - reply_version: int = 0x05, + method_version: int = _SOCKS5_VERSION, + reply_version: int = _SOCKS5_VERSION, eof_stage: Optional[str] = None, stall: bool = False, target: Optional[tuple[str, int]] = None, @@ -640,10 +642,10 @@ def _serve(self) -> None: conn.recv(1) return if self.eof_stage == "greeting": - conn.sendall(bytes((0x05,))) + conn.sendall(bytes((_SOCKS5_VERSION,))) return conn.sendall(bytes((self.method_version, self.method))) - if self.method == 0x02: + if self.method == _SOCKS5_AUTH_USER_PASS: auth_version, username_length = self._read(conn, 2) username = self._read(conn, username_length) password_length = self._read(conn, 1)[0] @@ -655,15 +657,15 @@ def _serve(self) -> None: + password ) if self.eof_stage == "auth": - conn.sendall(bytes((0x01,))) + conn.sendall(bytes((_SOCKS5_AUTH_VERSION,))) return - conn.sendall(bytes((0x01, self.auth_status))) + conn.sendall(bytes((_SOCKS5_AUTH_VERSION, self.auth_status))) if self.auth_status: return version, command, reserved, atyp = self._read(conn, 4) - if atyp == 0x01: + if atyp == _SOCKS5_ATYP_IPV4: addr = self._read(conn, 4) - elif atyp == 0x04: + elif atyp == _SOCKS5_ATYP_IPV6: addr = self._read(conn, 16) else: length = self._read(conn, 1) @@ -672,18 +674,24 @@ def _serve(self) -> None: bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) ) if self.eof_stage == "connect": - conn.sendall(bytes((0x05, 0x00))) + conn.sendall(bytes((_SOCKS5_VERSION, _SOCKS5_REPLY_SUCCESS))) return if self.target is not None: upstream = socket.create_connection(self.target, timeout=3) - if self.bound_atyp == 0x01: - bound = bytes((0x01,)) + socket.inet_aton("127.0.0.1") + bytes(2) - elif self.bound_atyp == 0x04: - bound = bytes((0x04,)) + socket.inet_pton(socket.AF_INET6, "::1") + bytes(2) + if self.bound_atyp == _SOCKS5_ATYP_IPV4: + bound = bytes((_SOCKS5_ATYP_IPV4,)) + socket.inet_aton("127.0.0.1") + bytes(2) + elif self.bound_atyp == _SOCKS5_ATYP_IPV6: + bound = ( + bytes((_SOCKS5_ATYP_IPV6,)) + + socket.inet_pton(socket.AF_INET6, "::1") + + bytes(2) + ) else: host = b"x" - bound = bytes((0x03, len(host))) + host + bytes(2) - conn.sendall(bytes((self.reply_version, self.reply, 0x00)) + bound + self.payload) + bound = bytes((_SOCKS5_ATYP_DOMAIN, len(host))) + host + bytes(2) + conn.sendall( + bytes((self.reply_version, self.reply, _SOCKS5_RESERVED)) + bound + self.payload + ) if self.payload: return if upstream is not None: @@ -746,25 +754,31 @@ async def _connect( return sock async def test_protocol_and_connection_errors(self) -> None: - for host, atyp in (("127.0.0.1", 1), ("2001:db8::1", 4), ("münich.example", 3)): + for host, atyp in ( + ("127.0.0.1", _SOCKS5_ATYP_IPV4), + ("2001:db8::1", _SOCKS5_ATYP_IPV6), + ("münich.example", _SOCKS5_ATYP_DOMAIN), + ): server = self._create_server() sock = await self._connect(server, (host, 27017)) sock.close() server.close() self.assertEqual(server.request[3], atyp) - server = self._create_server(method=2) + server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) sock = await self._connect(server, username="üser", password="påss") sock.close() server.close() self.assertIn("üser".encode(), server.auth) - for method in (1, 0xFF): + for method in (_SOCKS5_AUTH_GSSAPI_METHOD, _SOCKS5_AUTH_NO_ACCEPTABLE): server = self._create_server(method=method) with self.assertRaises(_Socks5Error): await self._connect(server) server.close() - server = self._create_server(method=2, auth_status=1) + server = self._create_server( + method=_SOCKS5_AUTH_USER_PASS, auth_status=_SOCKS5_AUTH_FAILURE_STATUS + ) with self.assertRaises(_Socks5Error): await self._connect(server, username="u", password="p") server.close()