diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 8e2214c680..ab3818ffdd 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -3778,6 +3778,36 @@ tasks: - pr - "9.0" + # Socks5 tests + - name: test-socks5-prose-noauth-nossl + commands: + - func: run server + vars: + TEST_NAME: socks5 + VERSION: "9.0" + TOPOLOGY: replica_set + AUTH: noauth + SSL: nossl + - func: run tests + vars: + TEST_NAME: socks5 + COVERAGE: "1" + tags: [socks5, pr] + - name: test-socks5-prose-noauth-ssl + commands: + - func: run server + vars: + TEST_NAME: socks5 + VERSION: "9.0" + TOPOLOGY: replica_set + AUTH: noauth + SSL: ssl + - func: run tests + vars: + TEST_NAME: socks5 + COVERAGE: "1" + tags: [socks5, pr] + # Standard tests - name: test-standard-v4.4-python3.12-sync-noauth-ssl-replica-set commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 67d069f438..5f53ae288a 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -631,6 +631,17 @@ buildvariants: TEST_NAME: sfp tags: [pr] + # Socks5 tests + - name: socks5-prose + tasks: + - name: .socks5 + display_name: SOCKS5 Prose Tests + run_on: + - rhel8.10-small + expansions: + UV_PYTHON: "3.11" + tags: [pr] + # Stable api tests - name: stable-api-require-v1-rhel8-auth tasks: diff --git a/.evergreen/remove-unimplemented-tests.sh b/.evergreen/remove-unimplemented-tests.sh index 7982dd6b21..fa4eda65da 100755 --- a/.evergreen/remove-unimplemented-tests.sh +++ b/.evergreen/remove-unimplemented-tests.sh @@ -46,9 +46,6 @@ find /$PYMONGO/test -type f -name 'pre-42-*.json' -delete rm $PYMONGO/test/csot/override-collection-timeoutMS.json rm $PYMONGO/test/csot/override-database-timeoutMS.json -# PYTHON-2943 - Socks5 Proxy Support -rm $PYMONGO/test/uri_options/proxy-options.json - # PYTHON-5517 - Avoid clearing the connection pool when the server connection rate limiter triggers rm $PYMONGO/test/discovery_and_monitoring/unified/backpressure-*.json diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 3589a81c86..99fcfe0a37 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -558,6 +558,19 @@ def create_coverage_report_variants(): return [create_variant(["coverage-report"], "Coverage Report", host=DEFAULT_HOST)] +def create_socks5_variants(): + variant = create_variant( + [".socks5"], + "SOCKS5 Prose Tests", + host=DEFAULT_HOST, + tags=["pr"], + expansions=dict(UV_PYTHON=MIN_PYTHON), + patchable=True, + ) + variant.name = "socks5-prose" + return [variant] + + def create_kms_variants(): tasks = [] tasks.append(EvgTaskRef(name="test-gcpkms", batchtime=BATCHTIME_DAY)) @@ -972,6 +985,28 @@ def create_fuzz_tasks(): return tasks +def create_socks5_tasks(): + tasks = [] + for ssl in ["nossl", "ssl"]: + server_vars = dict( + TEST_NAME="socks5", + VERSION=MAX_VERSION, + TOPOLOGY="replica_set", + AUTH="noauth", + SSL=ssl, + ) + commands = [ + FunctionCall(func="run server", vars=server_vars), + FunctionCall(func="run tests", vars=dict(TEST_NAME="socks5", COVERAGE="1")), + ] + tasks.append( + EvgTask( + name=f"test-socks5-prose-noauth-{ssl}", tags=["socks5", "pr"], commands=commands + ) + ) + return tasks + + def create_kms_tasks(): tasks = [] for kms_type in ["gcp", "azure"]: diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 9c3cdf13f5..2af8090a49 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -185,6 +185,9 @@ def handle_fuzz() -> None: def run() -> None: + if TEST_NAME == "socks5": + TEST_ARGS.extend(["test/test_socks5.py", "-k", "TestSocks5Prose"]) + # Add diagnostic for python version. print("Running with python", sys.version) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index bfa767e342..9e687c9cab 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -52,7 +52,6 @@ "pyopenssl": "ocsp", } - # Map the test name to test group. GROUP_MAP = dict(fuzz="fuzz", mockupdb="mockupdb", perf="perf") diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index f078f2f305..2fb836369c 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -72,6 +72,35 @@ def variant_selects(variant, task): return False +def test_socks5_tasks_use_standard_setup_and_execution(): + tasks = generate_config.create_socks5_tasks() + assert {task.name for task in tasks} == { + "test-socks5-prose-noauth-nossl", + "test-socks5-prose-noauth-ssl", + } + assert {task_vars(task)["SSL"] for task in tasks} == {"nossl", "ssl"} + for task in tasks: + assert {"socks5", "pr"} <= set(task.tags) + assert [command.func for command in task.commands] == ["run server", "run tests"] + assert task.commands[0].vars == dict( + TEST_NAME="socks5", + VERSION=generate_config_utils.MAX_VERSION, + TOPOLOGY="replica_set", + AUTH="noauth", + SSL=task_vars(task)["SSL"], + ) + assert task.commands[1].vars == dict(TEST_NAME="socks5", COVERAGE="1") + + +def test_socks5_variant_selects_both_prose_tasks(): + (variant,) = generate_config.create_socks5_variants() + assert variant.name == "socks5-prose" + assert variant.display_name == "SOCKS5 Prose Tests" + assert "pr" in variant.tags + assert variant.expansions["UV_PYTHON"] == generate_config_utils.MIN_PYTHON + assert all(variant_selects(variant, task) for task in generate_config.create_socks5_tasks()) + + def test_pr_tasks_pin_max_version(): """Task-level pins: pr tasks pin MAX_VERSION (the "latest" nightly is unstable); the explicit 4.4 min-deps pin is the exception. Variant-merged diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 5b7da0f5a3..01d83bb3a8 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -49,6 +49,7 @@ class Distro: "ocsp": "ocsp", "perf": "perf", "sfp": "sfp", + "socks5": "", "numpy": "", } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e683cbc5d1..1cfbf602b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -458,6 +458,16 @@ wheels (CPython 3.12-3.14). then `just run-tests`. Set `FUZZ_MAX_TIME` (seconds, default 300) to change the per-target budget. +## SOCKS5 proxy tests + +With a local replica set running and `MONGODB_URI` set to its connection string, +run `just setup-tests socks5`, then `just run-tests`, and finally +`just teardown-tests`. Setup requires the drivers-evergreen-tools checkout and +starts authenticated and unauthenticated proxies on ports 1080 and 1081. The +runner exercises the synchronous and asynchronous SOCKS5 prose tests; teardown +stops the proxies. For a TLS-enabled replica set, use +`just setup-tests socks5 --ssl` instead. + ## Testing time-dependent operations - `test.utils_shared.delay` - One can trigger an arbitrarily long-running operation on the server using this delay utility diff --git a/doc/changelog.rst b/doc/changelog.rst index dc0e697fdf..aa8553f650 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -4,6 +4,13 @@ Changelog Changes in Version 4.20.0 (2026/XX/XX) -------------------------------------- +- Added support for connecting to MongoDB and Client-Side Field Level + Encryption KMS servers through a SOCKS5 proxy via the ``proxyHost``, + ``proxyPort``, ``proxyUsername``, and ``proxyPassword`` URI and keyword + options on :class:`~pymongo.synchronous.mongo_client.MongoClient` and + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` + (`PYTHON-2943`_). SOCKS5 proxy options cannot be combined with + ``kms_connect_callback``. - The client handshake metadata now reports a detected coding agent in ``client.env.agent`` (`PYTHON-5929`_). @@ -35,6 +42,7 @@ Bug fixes The bug was fixed in Python 3.12.7 and Python 3.13.1. Users of Python 3.12 and 3.13 should upgrade to at least those versions (`PYTHON-6182`_). +.. _PYTHON-2943: https://jira.mongodb.org/browse/PYTHON-2943 .. _PYTHON-5929: https://jira.mongodb.org/browse/PYTHON-5929 .. _PYTHON-6172: https://jira.mongodb.org/browse/PYTHON-6172 .. _PYTHON-6149: https://jira.mongodb.org/browse/PYTHON-6149 diff --git a/pymongo/_kms_connect_shared.py b/pymongo/_kms_connect_shared.py index 634619dcdf..fc25950b47 100644 --- a/pymongo/_kms_connect_shared.py +++ b/pymongo/_kms_connect_shared.py @@ -32,10 +32,13 @@ import urllib.parse from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional from pymongo.errors import ConfigurationError +if TYPE_CHECKING: + from pymongo.pool_options import _ProxyOptions + @dataclass(frozen=True) class KMSConnectContext: @@ -71,6 +74,14 @@ class KMSConnectContext: KMSConnectCallback = Callable[[KMSConnectContext], socket.socket] +def _check_kms_proxy_options( + proxy_options: Optional[_ProxyOptions], + kms_connect_callback: Optional[Callable[[KMSConnectContext], Any]], +) -> None: + if proxy_options is not None and kms_connect_callback is not None: + raise ConfigurationError("Cannot specify kms_connect_callback with proxyHost") + + def _close_rejected_kms_socket(obj: Any) -> None: """Close a callback return value that failed validation, best effort. diff --git a/pymongo/asynchronous/_kms_connect.py b/pymongo/asynchronous/_kms_connect.py index ab74919beb..8b516c472d 100644 --- a/pymongo/asynchronous/_kms_connect.py +++ b/pymongo/asynchronous/_kms_connect.py @@ -63,7 +63,8 @@ async def _connect_kms( """Connect to a KMS host and perform the TLS handshake over the socket. Uses ``kms_connect_callback`` when one is provided, otherwise connects - directly, and always verifies against ``address`` (the KMS host). + using the configured SOCKS5 proxy or directly. Always verifies against + ``address`` (the KMS host). """ if kms_connect_callback is None: try: diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index e4f91294a4..4cff2ec34f 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -53,7 +53,7 @@ from bson.codec_options import CodecOptions from bson.raw_bson import DEFAULT_RAW_BSON_OPTIONS, RawBSONDocument, _inflate_bson from pymongo import _csot, _op_id -from pymongo._kms_connect_shared import AsyncKMSConnectCallback +from pymongo._kms_connect_shared import AsyncKMSConnectCallback, _check_kms_proxy_options from pymongo.asynchronous._kms_connect import _KMS_CONNECT_TIMEOUT, _connect_kms from pymongo.asynchronous.collection import AsyncCollection from pymongo.asynchronous.cursor import AsyncCursor @@ -102,6 +102,8 @@ if TYPE_CHECKING: from pymongocrypt.mongocrypt import MongoCryptKmsContext + from pymongo.pool_options import _ProxyOptions + _IS_SYNC = False @@ -123,6 +125,7 @@ def __init__( key_vault_coll: AsyncCollection[_DocumentTypeArg], mongocryptd_client: Optional[AsyncMongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, + proxy_options: Optional[_ProxyOptions] = None, ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -140,6 +143,7 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client + self._proxy_options = proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) @@ -182,6 +186,7 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: connect_timeout=connect_timeout, socket_timeout=connect_timeout, ssl_context=ctx, + proxy_options=self._proxy_options, ) try: conn = await _connect_kms( @@ -360,6 +365,9 @@ def __init__(self, client: AsyncMongoClient[_DocumentTypeArg], opts: AutoEncrypt :param client: The encrypted AsyncMongoClient. :param opts: The encrypted client's :class:`AutoEncryptionOpts`. """ + _check_kms_proxy_options( + client.options.pool_options._proxy_options, opts._kms_connect_callback + ) if opts._schema_map is None: schema_map = None else: @@ -387,6 +395,7 @@ def _get_internal_client( encrypter._internal_client = internal_client return internal_client + # KMS requests use the encrypted client's SOCKS5 proxy options, not the key vault client's. if opts._key_vault_client is not None: key_vault_client = opts._key_vault_client else: @@ -409,6 +418,7 @@ def _get_internal_client( key_vault_coll, # type:ignore[arg-type] mongocryptd_client, opts, + proxy_options=client.options.pool_options._proxy_options, ) self._auto_encrypter = AsyncAutoEncrypter( io_callbacks, @@ -572,7 +582,8 @@ def __init__( collection. Data keys are protected with encryption by a KMS provider. :param key_vault_client: An AsyncMongoClient connected to a MongoDB cluster - containing the `key_vault_namespace` collection. + containing the `key_vault_namespace` collection. Its SOCKS5 proxy + options are also used for KMS requests. :param codec_options: An instance of :class:`~bson.codec_options.CodecOptions` to use when encoding a value for encryption and decoding the decrypted BSON value. This @@ -604,8 +615,11 @@ def __init__( deadline and closes any socket the callback yields later. For an ordinary HTTP proxy, pass :class:`~pymongo.encryption_options.AsyncHTTPProxyKMSConnect`. - Defaults to ``None``, meaning the driver connects to KMS hosts - directly. + Defaults to ``None``, meaning the driver uses the SOCKS5 proxy + configured on ``key_vault_client``, or connects directly if none + is configured. Cannot be combined with ``proxyHost`` on + ``key_vault_client``; doing so raises + :class:`~pymongo.errors.ConfigurationError`. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. @@ -639,6 +653,9 @@ def __init__( f"AsyncMongoClient required but given {type(key_vault_client).__name__}" ) + _check_kms_proxy_options( + key_vault_client.options.pool_options._proxy_options, kms_connect_callback + ) self._kms_providers = kms_providers self._key_vault_namespace = key_vault_namespace self._key_vault_client = key_vault_client @@ -656,7 +673,11 @@ def __init__( ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( - None, key_vault_coll, None, opts + None, + key_vault_coll, + None, + opts, + proxy_options=key_vault_client.options.pool_options._proxy_options, ) self._encryption = AsyncExplicitEncrypter( self._io_callbacks, diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index e1837c05ed..b6c782b3de 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -107,6 +107,7 @@ UpdateOne, _Op, ) +from pymongo.pool_options import _ProxyOptions from pymongo.read_preferences import ReadPreference, _ServerMode from pymongo.results import ClientBulkWriteResult from pymongo.server_description import ServerDescription @@ -159,6 +160,8 @@ Coroutine[Any, Any, T], ] +_REDACTED_REPR_OPTS = frozenset(["username", "password", "proxyusername", "proxypassword"]) + _IS_SYNC = False _WriteOp = Union[ @@ -338,6 +341,18 @@ def __init__( connecting a new socket to a server before concluding the server is unavailable. ``0`` or ``None`` means no timeout. Defaults to ``20000`` (20 seconds). + - `proxyHost` (string): The hostname or IP address of a SOCKS5 proxy. + IPv6 literals may be enclosed in brackets. Defaults to no proxy. + Routes MongoDB and automatic encryption KMS connections through + the proxy. Cannot be combined with ``kms_connect_callback`` in + ``auto_encryption_opts``; doing so raises + :class:`~pymongo.errors.ConfigurationError`. + - `proxyPort` (integer): The TCP port of the SOCKS5 proxy. Defaults + to ``1080`` when ``proxyHost`` is specified. + - `proxyUsername` (string): The username for SOCKS5 username/password + authentication. Must be specified together with ``proxyPassword``. + - `proxyPassword` (string): The password for SOCKS5 username/password + authentication. Must be specified together with ``proxyUsername``. - `server_selector`: (callable or None) Optional, user-provided function that augments server selection rules. The function should accept as an argument a list of @@ -671,6 +686,10 @@ def validator(host: str) -> bool: .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.20 + Added the ``proxyHost``, ``proxyPort``, ``proxyUsername``, and + ``proxyPassword`` keyword arguments and URI options. + .. versionchanged:: 4.19 Added the ``srv_host_validator`` keyword argument. @@ -878,6 +897,7 @@ def validator(host: str) -> bool: warn=True, normalize=False, srv_max_hosts=srv_max_hosts, + validate_proxy=False, ) is_srv = entity.startswith(SRV_SCHEME) self._seeds.update(res["nodelist"]) @@ -1003,6 +1023,7 @@ async def _resolve_srv(self) -> None: srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, srv_host_validator=srv_host_validator, + validate_proxy=False, ) seeds.update(res["nodelist"]) opts = res["options"] @@ -1378,12 +1399,15 @@ def options(self) -> ClientOptions: """ return self._options - def eq_props(self) -> tuple[tuple[_Address, ...], Optional[str], Optional[str], str]: + def eq_props( + self, + ) -> tuple[tuple[_Address, ...], Optional[str], Optional[str], str, Optional[_ProxyOptions]]: return ( tuple(sorted(self._resolve_srv_info["seeds"])), self._options.replica_set_name, self._resolve_srv_info["fqdn"], self._resolve_srv_info["srv_service_name"], + self._options.pool_options._proxy_options, ) def __eq__(self, other: Any) -> bool: @@ -1432,7 +1456,7 @@ def option_repr(option: str, value: Any) -> str: options.extend( option_repr(key, self._options._options[key]) for key in self._options._options - if key not in set(self._constructor_args) and key != "username" and key != "password" + if key not in set(self._constructor_args) and key not in _REDACTED_REPR_OPTS ) return ", ".join(options) diff --git a/pymongo/asynchronous/socks5_support.py b/pymongo/asynchronous/socks5_support.py new file mode 100644 index 0000000000..90c2ca3c27 --- /dev/null +++ b/pymongo/asynchronous/socks5_support.py @@ -0,0 +1,136 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Asynchronous SOCKS5 proxy connection establishment.""" + +from __future__ import annotations + +import socket +from typing import TYPE_CHECKING, Optional + +from pymongo.pool_shared import _configure_tcp_socket, _new_tcp_socket +from pymongo.socks5_shared import ( + _SOCKS5_ATYP_DOMAIN, + _SOCKS5_AUTH_USER_PASS, + _async_proxy_connect, + _async_proxy_getaddrinfo, + _async_socks5_recv, + _async_socks5_sendall, + _proxy_deadline, + _proxy_remaining, + _proxy_socket_timeout, + _socks5_auth_request, + _socks5_bound_address_size, + _socks5_check_auth_reply, + _socks5_check_connect_reply, + _socks5_check_greeting_reply, + _socks5_greeting, + _socks5_request, + _Socks5Error, + _Socks5Timeout, +) + +if TYPE_CHECKING: + from pymongo.pool_options import PoolOptions, _ProxyOptions + from pymongo.typings import _Address + +_IS_SYNC = False + + +async def _async_socks5_read_exact( + sock: socket.socket, size: int, deadline: Optional[float] +) -> bytes: + data = bytearray() + while len(data) < size: + chunk = await _async_socks5_recv(sock, size - len(data), deadline) + if not chunk: + raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") + data.extend(chunk) + return bytes(data) + + +async def _async_socks5_handshake( + sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] +) -> None: + greeting, allowed = _socks5_greeting(proxy) + await _async_socks5_sendall(sock, greeting, deadline) + method = _socks5_check_greeting_reply( + await _async_socks5_read_exact(sock, 2, deadline), allowed + ) + if method == _SOCKS5_AUTH_USER_PASS: + await _async_socks5_sendall(sock, _socks5_auth_request(proxy), deadline) + _socks5_check_auth_reply(await _async_socks5_read_exact(sock, 2, deadline)) + await _async_socks5_sendall(sock, _socks5_request(address), deadline) + atyp = _socks5_check_connect_reply(await _async_socks5_read_exact(sock, 4, deadline)) + domain_length = ( + (await _async_socks5_read_exact(sock, 1, deadline))[0] if atyp == _SOCKS5_ATYP_DOMAIN else 0 + ) + bound_size = _socks5_bound_address_size(atyp, domain_length) + await _async_socks5_read_exact(sock, bound_size, deadline) + + +async def _async_create_proxy_connection( + address: _Address, options: PoolOptions, proxy: _ProxyOptions +) -> socket.socket: + deadline = _proxy_deadline(options) + family = socket.AF_INET + if socket.has_ipv6 and proxy.host != "localhost": + family = socket.AF_UNSPEC + try: + results = await _async_proxy_getaddrinfo(proxy, family, deadline) + except TimeoutError as exc: + raise _Socks5Timeout("SOCKS5 proxy timed out") from exc + except OSError as exc: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc + + err: Optional[OSError] = None + for af, socktype, proto, _, sockaddr in results: + sock = _new_tcp_socket(af, socktype, proto) + connected = False + try: + _configure_tcp_socket(sock, non_blocking=not _IS_SYNC) + await _async_proxy_connect(sock, sockaddr, deadline) + connected = True + await _async_socks5_handshake(sock, address, proxy, deadline) + sock.settimeout(_proxy_socket_timeout(options)) + return sock + except TimeoutError as exc: + sock.close() + if connected: + raise _Socks5Timeout("SOCKS5 proxy timed out") from exc + try: + _proxy_remaining(deadline) + except _Socks5Timeout as timeout: + raise timeout from exc + err = _Socks5Timeout("SOCKS5 proxy timed out") + err.__cause__ = exc + except _Socks5Error: + sock.close() + raise + except OSError as exc: + sock.close() + err = exc + # A connected proxy's handshake failure must not try a different + # resolved proxy address; the failure is specific to the endpoint. + if connected: + raise _Socks5Error(f"SOCKS5 handshake failed: {exc}") from exc + except BaseException: + sock.close() + raise + + if isinstance(err, socket.timeout): + raise err + if err is not None: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err + raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") diff --git a/pymongo/asynchronous/uri_parser.py b/pymongo/asynchronous/uri_parser.py index a904c4d283..c285c825ed 100644 --- a/pymongo/asynchronous/uri_parser.py +++ b/pymongo/asynchronous/uri_parser.py @@ -137,6 +137,7 @@ async def _parse_srv( srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, srv_host_validator: Optional[Callable[[str], bool]] = None, + validate_proxy: bool = True, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -199,7 +200,7 @@ async def _parse_srv( else: nodes = split_hosts(hosts, default_port=default_port) - _check_options(nodes, options) + _check_options(nodes, options, validate_proxy=validate_proxy) return { "nodelist": nodes, diff --git a/pymongo/client_options.py b/pymongo/client_options.py index b42635ef2c..ed95e00646 100644 --- a/pymongo/client_options.py +++ b/pymongo/client_options.py @@ -27,7 +27,7 @@ from pymongo.compression_support import CompressionSettings from pymongo.errors import ConfigurationError from pymongo.monitoring import _EventListener, _EventListeners -from pymongo.pool_options import PoolOptions +from pymongo.pool_options import _DEFAULT_PROXY_PORT, PoolOptions, _ProxyOptions from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ( _ServerMode, @@ -174,6 +174,17 @@ def _parse_pool_options( options.get("compressors", []), options.get("zlibcompressionlevel", -1) ) ssl_context, tls_allow_invalid_hostnames = _parse_ssl_options(options, is_sync) + proxy_host = options.get("proxyhost") + proxy_options = ( + _ProxyOptions( + proxy_host, + options.get("proxyport") or _DEFAULT_PROXY_PORT, + options.get("proxyusername"), + options.get("proxypassword"), + ) + if proxy_host is not None + else None + ) load_balanced = options.get("loadbalanced") max_connecting = options.get("maxconnecting", common.MAX_CONNECTING) return PoolOptions( @@ -194,6 +205,7 @@ def _parse_pool_options( load_balanced=load_balanced, credentials=credentials, is_sync=is_sync, + proxy_options=proxy_options, ) diff --git a/pymongo/common.py b/pymongo/common.py index 224b762fa5..38cdae4eeb 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -18,6 +18,8 @@ from __future__ import annotations import datetime +import ipaddress +import unicodedata import warnings from collections import OrderedDict, abc from collections.abc import Callable, Iterator, Mapping, MutableMapping, Sequence @@ -148,6 +150,11 @@ # Auth mechanism properties that must raise an error instead of warning if they invalidate. _MECH_PROP_MUST_RAISE = ["CANONICALIZE_HOST_NAME"] +# URI options that must raise an error instead of warning if they invalidate, +# so a malformed proxy option never silently falls back to a direct connection. +_URI_OPTS_MUST_RAISE = frozenset(["proxyhost", "proxyport", "proxyusername", "proxypassword"]) +_SOCKS5_MAX_STR_LEN = 255 + def partition_node(node: str) -> tuple[str, int]: """Split a host:port string into (host, int(port)) pair.""" @@ -227,6 +234,66 @@ def validate_non_negative_integer(option: str, value: Any) -> int: return val +def validate_proxy_host(option: str, value: Any) -> Optional[str]: + """Validate and normalize a SOCKS5 proxy host. + + :raises TypeError: If ``value`` is not a string or ``None``. + :raises ValueError: If ``value`` is an invalid host. + """ + if value is None: + return None + host = validate_string(option, value) + if not host: + raise ValueError(f"{option} cannot be an empty string") + if any(char.isspace() or unicodedata.category(char) == "Cc" for char in host): + raise ValueError(f"{option} must not contain whitespace or control characters") + if host.startswith("[") or host.endswith("]"): + if not (host.startswith("[") and host.endswith("]")): + raise ValueError(f"{option} has invalid bracket syntax") + literal = host[1:-1] + try: + ipaddress.IPv6Address(literal.split("%", 1)[0]) + except ipaddress.AddressValueError: + raise ValueError(f"{option} must contain a bracketed IPv6 address") from None + return literal + if "[" in host or "]" in host or "/" in host: + raise ValueError(f"{option} has invalid host syntax") + if ":" in host: + try: + ipaddress.IPv6Address(host.split("%", 1)[0]) + except ipaddress.AddressValueError: + raise ValueError(f"{option} must not include a port") from None + return host + + +def validate_proxy_port(option: str, value: Any) -> Optional[int]: + """Validate a SOCKS5 proxy TCP port. + + :raises TypeError: If ``value`` is not an integer or string. + :raises ValueError: If ``value`` is not an integer or is outside the valid port range. + """ + if value is None: + return None + if isinstance(value, bool): + raise TypeError(f"Wrong type for {option}, value must be an integer, not bool") + port = validate_integer(option, value) + if not 1 <= port <= 65535: + raise ValueError(f"{option} must be an integer between 1 and 65535") + return port + + +def validate_proxy_credential(option: str, value: Any) -> Optional[str]: + """Validate a SOCKS5 username or password and its UTF-8 byte length. + + :raises TypeError: If ``value`` is not a string or ``None``. + :raises ValueError: If the UTF-8 byte length exceeds the SOCKS5 limit. + """ + credential = validate_string_or_none(option, value) + if credential is not None and len(credential.encode("utf-8")) > _SOCKS5_MAX_STR_LEN: + raise ValueError(f"{option} must be no more than {_SOCKS5_MAX_STR_LEN} UTF-8 bytes") + return credential + + def validate_readable(option: str, value: Any) -> Optional[str]: """Validates that 'value' is file-like and readable.""" if value is None: @@ -724,6 +791,10 @@ def validate_server_monitoring_mode(option: str, value: str) -> str: "maxconnecting": validate_positive_integer, "maxpoolsize": validate_non_negative_integer_or_none, "maxstalenessseconds": validate_max_staleness, + "proxyhost": validate_proxy_host, + "proxyport": validate_proxy_port, + "proxyusername": validate_proxy_credential, + "proxypassword": validate_proxy_credential, "readconcernlevel": validate_string_or_none, "readpreference": validate_read_preference_mode, "readpreferencetags": validate_read_preference_tags, @@ -888,7 +959,7 @@ def get_setter_key(x: str) -> str: validator = _get_validator(opt, URI_OPTIONS_VALIDATOR_MAP, normed_key=normed_key) validated = validator(opt, value) except (ValueError, TypeError, ConfigurationError) as exc: - if ( + if normed_key in _URI_OPTS_MUST_RAISE or ( normed_key == "authmechanismproperties" and any(p in str(exc) for p in _MECH_PROP_MUST_RAISE) and "is not a supported auth mechanism property" not in str(exc) diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index f02b28e9b8..42bb16c826 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -228,7 +228,10 @@ def __init__( :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` and a regular function for :class:`~pymongo.synchronous.mongo_client.MongoClient`. Defaults - to ``None``, meaning the driver connects to KMS hosts directly. + to ``None``, meaning the driver uses the encrypted client's SOCKS5 + proxy, or connects directly if none is configured. Cannot be + combined with ``proxyHost`` on the encrypted client; doing so + raises :class:`~pymongo.errors.ConfigurationError`. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. diff --git a/pymongo/pool_options.py b/pymongo/pool_options.py index 45a9adf5da..b79a344822 100644 --- a/pymongo/pool_options.py +++ b/pymongo/pool_options.py @@ -25,6 +25,7 @@ import sys from collections.abc import MutableMapping from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, Optional @@ -102,6 +103,7 @@ RUNTIME_NAME_DOCKER = "docker" ORCHESTRATOR_NAME_K8S = "kubernetes" +_DEFAULT_PROXY_PORT = 1080 def get_container_env_info() -> dict[str, str]: @@ -369,6 +371,16 @@ def _truncate_metadata(metadata: MutableMapping[str, Any]) -> None: "foo".encode("idna") +@dataclass(frozen=True, slots=True) +class _ProxyOptions: + """Immutable SOCKS5 proxy connection settings.""" + + host: str + port: int = _DEFAULT_PROXY_PORT + username: Optional[str] = None + password: Optional[str] = field(default=None, repr=False) + + class PoolOptions: """Read only connection pool options for an AsyncMongoClient/MongoClient. @@ -398,6 +410,7 @@ class PoolOptions: "__metadata_lock", "__min_pool_size", "__pause_enabled", + "__proxy_options", "__server_api", "__socket_timeout", "__ssl_context", @@ -425,6 +438,7 @@ def __init__( load_balanced: Optional[bool] = None, credentials: Optional[MongoCredential] = None, is_sync: Optional[bool] = True, + proxy_options: Optional[_ProxyOptions] = None, ): self.__max_pool_size = max_pool_size self.__min_pool_size = min_pool_size @@ -443,6 +457,7 @@ def __init__( self.__server_api = server_api self.__load_balanced = load_balanced self.__credentials = credentials + self.__proxy_options = proxy_options self.__metadata = copy.deepcopy(_METADATA) self.__appended_drivers: set[DriverInfo] = set() # Only the synchronous client can append metadata from multiple threads. @@ -607,6 +622,11 @@ def wait_queue_timeout(self) -> Optional[int]: """ return self.__wait_queue_timeout + @property + def _proxy_options(self) -> Optional[_ProxyOptions]: + """SOCKS5 proxy settings or None.""" + return self.__proxy_options + @property def _ssl_context(self) -> Optional[SSLContext]: """An SSLContext instance or None.""" diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 48a0a27bf1..27240b442a 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -66,6 +66,7 @@ from pymongo.pool_options import PoolOptions from pymongo.server_api import _add_to_command from pymongo.socket_checker import SocketChecker +from pymongo.socks5_shared import _Socks5Error, _Socks5Timeout from pymongo.ssl_support import SSL_EOF_ERRORS, PYSSLError, SSLError, _has_sni SSLErrors = (PYSSLError, SSLError) @@ -340,8 +341,11 @@ def _handle_connection_error(self, error: BaseException) -> None: if self.is_sdam or type(error) not in (AutoReconnect, NetworkTimeout): return assert isinstance(error, AutoReconnect) # Appease type checker. - # If the original error was a DNS, certificate, or SSL error, ignore it. - if isinstance(error.__cause__, (_CertificateError, SSLErrors, socket.gaierror)): + # Ignore DNS, certificate, SSL, and SOCKS5 proxy errors. + if isinstance( + error.__cause__, + (_CertificateError, SSLErrors, socket.gaierror, _Socks5Error, _Socks5Timeout), + ): # End of file errors are excluded, because the server may have disconnected # during the handshake. if not isinstance(error.__cause__, SSL_EOF_ERRORS): @@ -593,6 +597,26 @@ class PoolState: CLOSED = 3 +def _new_tcp_socket(af: int, socktype: int, proto: int) -> socket.socket: + # SOCK_CLOEXEC was added in CPython 3.2 and is not available on all + # platforms or kernels. PEP 446 makes descriptors non-inheritable by + # default starting in CPython 3.4. + try: + sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) + except OSError: + sock = socket.socket(af, socktype, proto) + _set_non_inheritable_non_atomic(sock.fileno()) + return sock + + +def _configure_tcp_socket(sock: socket.socket, *, non_blocking: bool = False) -> None: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + _set_keepalive_times(sock) + if non_blocking: + sock.setblocking(False) + + async def _async_create_connection(address: _Address, options: PoolOptions) -> socket.socket: """Given (host, port) and PoolOptions, connect and return a raw socket object. @@ -602,6 +626,12 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s """ host, port = address + if options._proxy_options is not None: + # Import locally to avoid a circular import; socks5_support imports pool_shared. + from pymongo.asynchronous.socks5_support import _async_create_proxy_connection + + return await _async_create_proxy_connection(address, options, options._proxy_options) + # Check if dealing with a unix domain socket if host.endswith(".sock"): if not hasattr(socket, "AF_UNIX"): @@ -627,29 +657,15 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s err = None for res in await _getaddrinfo(host, port, family=family, type=socket.SOCK_STREAM): af, socktype, proto, dummy, sa = res - # SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited - # number of platforms (newer Linux and *BSD). Starting with CPython 3.4 - # all file descriptors are created non-inheritable. See PEP 446. + sock = _new_tcp_socket(af, socktype, proto) try: - sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) - except OSError: - # Can SOCK_CLOEXEC be defined even if the kernel doesn't support - # it? - sock = socket.socket(af, socktype, proto) - # Fallback when SOCK_CLOEXEC isn't available. - _set_non_inheritable_non_atomic(sock.fileno()) - try: - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + _configure_tcp_socket(sock, non_blocking=True) # CSOT: apply timeout to socket connect. timeout = _csot.remaining() if timeout is None: timeout = options.connect_timeout elif timeout <= 0: raise socket.timeout("timed out") - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - _set_keepalive_times(sock) - # Socket needs to be non-blocking during connection to not block the event loop - sock.setblocking(False) await asyncio.wait_for( asyncio.get_running_loop().sock_connect(sock, sa), timeout=timeout ) @@ -849,6 +865,12 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket """ host, port = address + if options._proxy_options is not None: + # Import locally to avoid a circular import; socks5_support imports pool_shared. + from pymongo.synchronous.socks5_support import _create_proxy_connection + + return _create_proxy_connection(address, options, options._proxy_options) + # Check if dealing with a unix domain socket if host.endswith(".sock"): if not hasattr(socket, "AF_UNIX"): @@ -873,19 +895,9 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket err = None for res in socket.getaddrinfo(host, port, family=family, type=socket.SOCK_STREAM): # type: ignore[attr-defined, unused-ignore] af, socktype, proto, dummy, sa = res - # SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited - # number of platforms (newer Linux and *BSD). Starting with CPython 3.4 - # all file descriptors are created non-inheritable. See PEP 446. - try: - sock = socket.socket(af, socktype | getattr(socket, "SOCK_CLOEXEC", 0), proto) - except OSError: - # Can SOCK_CLOEXEC be defined even if the kernel doesn't support - # it? - sock = socket.socket(af, socktype, proto) - # Fallback when SOCK_CLOEXEC isn't available. - _set_non_inheritable_non_atomic(sock.fileno()) + sock = _new_tcp_socket(af, socktype, proto) try: - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + _configure_tcp_socket(sock) # CSOT: apply timeout to socket connect. timeout = _csot.remaining() if timeout is None: @@ -893,8 +905,6 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket elif timeout <= 0: raise socket.timeout("timed out") sock.settimeout(timeout) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - _set_keepalive_times(sock) sock.connect(sa) return sock except OSError as e: diff --git a/pymongo/socks5_shared.py b/pymongo/socks5_shared.py new file mode 100644 index 0000000000..f5b4991d44 --- /dev/null +++ b/pymongo/socks5_shared.py @@ -0,0 +1,243 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SOCKS5 protocol utilities and socket helpers shared by both APIs.""" + +from __future__ import annotations + +import asyncio +import ipaddress +import socket +import time +from typing import TYPE_CHECKING, Optional, cast + +from pymongo import _csot +from pymongo.asynchronous.helpers import _getaddrinfo +from pymongo.common import _SOCKS5_MAX_STR_LEN + +if TYPE_CHECKING: + from pymongo.pool_options import PoolOptions, _ProxyOptions + from pymongo.typings import _Address + +_SOCKS5_VERSION = 0x05 +_SOCKS5_AUTH_VERSION = 0x01 +_SOCKS5_AUTH_NO_AUTH = 0x00 +_SOCKS5_AUTH_USER_PASS = 0x02 +_SOCKS5_AUTH_NO_ACCEPTABLE = 0xFF +_SOCKS5_AUTH_SUCCESS = 0x00 +_SOCKS5_RESERVED = 0x00 +_SOCKS5_CMD_CONNECT = 0x01 +_SOCKS5_REPLY_SUCCESS = 0x00 +_SOCKS5_REPLY_GENERAL_FAILURE = 0x01 +_SOCKS5_REPLY_RULESET_DENIED = 0x02 +_SOCKS5_REPLY_NETWORK_UNREACHABLE = 0x03 +_SOCKS5_REPLY_HOST_UNREACHABLE = 0x04 +_SOCKS5_REPLY_CONNECTION_REFUSED = 0x05 +_SOCKS5_REPLY_TTL_EXPIRED = 0x06 +_SOCKS5_REPLY_COMMAND_UNSUPPORTED = 0x07 +_SOCKS5_REPLY_ADDRESS_UNSUPPORTED = 0x08 +_SOCKS5_ATYP_IPV4 = 0x01 +_SOCKS5_ATYP_DOMAIN = 0x03 +_SOCKS5_ATYP_IPV6 = 0x04 + +_SOCKS5_REPLY_REASONS = { + _SOCKS5_REPLY_GENERAL_FAILURE: "general SOCKS server failure", + _SOCKS5_REPLY_RULESET_DENIED: "connection not allowed by ruleset", + _SOCKS5_REPLY_NETWORK_UNREACHABLE: "network unreachable", + _SOCKS5_REPLY_HOST_UNREACHABLE: "host unreachable", + _SOCKS5_REPLY_CONNECTION_REFUSED: "connection refused", + _SOCKS5_REPLY_TTL_EXPIRED: "TTL expired", + _SOCKS5_REPLY_COMMAND_UNSUPPORTED: "command not supported", + _SOCKS5_REPLY_ADDRESS_UNSUPPORTED: "address type not supported", +} + +_ProxyAddress = tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes] +_ProxyAddressInfo = tuple[socket.AddressFamily, socket.SocketKind, int, str, _ProxyAddress] + + +class _Socks5Error(OSError): + """A SOCKS5 proxy connection or protocol failure.""" + + +class _Socks5Timeout(socket.timeout): + """A timeout while connecting through a SOCKS5 proxy.""" + + +def _proxy_deadline(options: PoolOptions) -> Optional[float]: + timeout = _proxy_socket_timeout(options) + if timeout is None: + return None + return time.monotonic() + timeout + + +def _proxy_socket_timeout(options: PoolOptions) -> Optional[float]: + timeout = _csot.remaining() + if timeout is None: + timeout = options.connect_timeout + if timeout is not None and timeout <= 0: + raise _Socks5Timeout("SOCKS5 proxy timed out") + return timeout + + +def _proxy_remaining(deadline: Optional[float]) -> Optional[float]: + if deadline is None: + return None + remaining = deadline - time.monotonic() + if remaining <= 0: + raise _Socks5Timeout("SOCKS5 proxy timed out") + return remaining + + +def _socks5_bound_address_size(atyp: int, domain_length: int = 0) -> int: + if atyp == _SOCKS5_ATYP_IPV4: + return 6 + if atyp == _SOCKS5_ATYP_IPV6: + return 18 + return domain_length + 2 + + +def _socks5_greeting(proxy: _ProxyOptions) -> tuple[bytes, tuple[int, ...]]: + if proxy.username is not None and proxy.password is not None: + return bytes( + ( + _SOCKS5_VERSION, + 2, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + ), ( + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_USER_PASS, + ) + return bytes((_SOCKS5_VERSION, 1, _SOCKS5_AUTH_NO_AUTH)), (_SOCKS5_AUTH_NO_AUTH,) + + +def _socks5_check_greeting_reply(data: bytes, allowed: tuple[int, ...]) -> int: + version, method = data + if version != _SOCKS5_VERSION: + raise _Socks5Error(f"SOCKS5 initial handshake failed: unexpected version {version:#04x}") + if method not in allowed: + if method == _SOCKS5_AUTH_NO_ACCEPTABLE: + raise _Socks5Error( + "SOCKS5 initial handshake failed: no acceptable authentication methods" + ) + raise _Socks5Error( + f"SOCKS5 initial handshake failed: unexpected authentication method {method:#04x}" + ) + return method + + +def _socks5_auth_request(proxy: _ProxyOptions) -> bytes: + assert proxy.username is not None + assert proxy.password is not None + username = proxy.username.encode("utf-8") + password = proxy.password.encode("utf-8") + return ( + bytes((_SOCKS5_AUTH_VERSION, len(username))) + username + bytes((len(password),)) + password + ) + + +def _socks5_check_auth_reply(data: bytes) -> None: + auth_version, status = data + if auth_version != _SOCKS5_AUTH_VERSION or status != _SOCKS5_AUTH_SUCCESS: + raise _Socks5Error("SOCKS5 authentication failed") + + +def _socks5_request(address: _Address) -> bytes: + host, port = address + port = cast(int, port) + try: + ip = ipaddress.ip_address(host.split("%", 1)[0]) + except ValueError: + try: + domain = host.encode("ascii") if host.isascii() else host.encode("idna") + except UnicodeError as exc: + raise _Socks5Error(f"SOCKS5 connect failed: invalid domain name: {exc}") from exc + if not 1 <= len(domain) <= _SOCKS5_MAX_STR_LEN: + raise _Socks5Error("SOCKS5 connect failed: invalid domain name length") from None + encoded_host = bytes((_SOCKS5_ATYP_DOMAIN, len(domain))) + domain + else: + encoded_host = ( + bytes((_SOCKS5_ATYP_IPV4 if ip.version == 4 else _SOCKS5_ATYP_IPV6,)) + ip.packed + ) + return ( + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) + + encoded_host + + port.to_bytes(2, "big") + ) + + +def _socks5_check_connect_reply(data: bytes) -> int: + version, reply, reserved, atyp = data + if version != _SOCKS5_VERSION: + raise _Socks5Error(f"SOCKS5 connect failed: unexpected version {version:#04x}") + if reserved != _SOCKS5_RESERVED: + raise _Socks5Error(f"SOCKS5 connect failed: invalid reserved byte {reserved:#04x}") + if reply != _SOCKS5_REPLY_SUCCESS: + reason = _SOCKS5_REPLY_REASONS.get(reply, f"unknown error {reply:#04x}") + raise _Socks5Error(f"SOCKS5 connect failed: {reason}") + if atyp not in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): + raise _Socks5Error(f"SOCKS5 connect failed: invalid address type {atyp:#04x}") + return atyp + + +def _socks5_sendall(sock: socket.socket, message: bytes, deadline: Optional[float]) -> None: + sock.settimeout(_proxy_remaining(deadline)) + sock.sendall(message) + + +async def _async_socks5_sendall( + sock: socket.socket, message: bytes, deadline: Optional[float] +) -> None: + loop = asyncio.get_running_loop() + async with asyncio.timeout(_proxy_remaining(deadline)): + await loop.sock_sendall(sock, message) + + +def _socks5_recv(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: + sock.settimeout(_proxy_remaining(deadline)) + return sock.recv(size) + + +async def _async_socks5_recv(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: + loop = asyncio.get_running_loop() + async with asyncio.timeout(_proxy_remaining(deadline)): + return await loop.sock_recv(sock, size) + + +def _proxy_getaddrinfo( + proxy: _ProxyOptions, family: int, deadline: Optional[float] +) -> list[_ProxyAddressInfo]: + _proxy_remaining(deadline) + return socket.getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) + + +async def _async_proxy_getaddrinfo( + proxy: _ProxyOptions, family: int, deadline: Optional[float] +) -> list[_ProxyAddressInfo]: + async with asyncio.timeout(_proxy_remaining(deadline)): + return await _getaddrinfo(proxy.host, proxy.port, family=family, type=socket.SOCK_STREAM) + + +def _proxy_connect(sock: socket.socket, address: _ProxyAddress, deadline: Optional[float]) -> None: + sock.settimeout(_proxy_remaining(deadline)) + sock.connect(address) + + +async def _async_proxy_connect( + sock: socket.socket, address: _ProxyAddress, deadline: Optional[float] +) -> None: + loop = asyncio.get_running_loop() + async with asyncio.timeout(_proxy_remaining(deadline)): + await loop.sock_connect(sock, address) diff --git a/pymongo/synchronous/_kms_connect.py b/pymongo/synchronous/_kms_connect.py index 78d8119557..7d53a188d9 100644 --- a/pymongo/synchronous/_kms_connect.py +++ b/pymongo/synchronous/_kms_connect.py @@ -63,7 +63,8 @@ def _connect_kms( """Connect to a KMS host and perform the TLS handshake over the socket. Uses ``kms_connect_callback`` when one is provided, otherwise connects - directly, and always verifies against ``address`` (the KMS host). + using the configured SOCKS5 proxy or directly. Always verifies against + ``address`` (the KMS host). """ if kms_connect_callback is None: try: diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 1626c16f58..8713697b36 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -52,7 +52,7 @@ from bson.codec_options import CodecOptions from bson.raw_bson import DEFAULT_RAW_BSON_OPTIONS, RawBSONDocument, _inflate_bson from pymongo import _csot, _op_id -from pymongo._kms_connect_shared import KMSConnectCallback +from pymongo._kms_connect_shared import KMSConnectCallback, _check_kms_proxy_options from pymongo.daemon import _spawn_daemon from pymongo.encryption_options import ( AutoEncryptionOpts, @@ -101,6 +101,8 @@ if TYPE_CHECKING: from pymongocrypt.mongocrypt import MongoCryptKmsContext + from pymongo.pool_options import _ProxyOptions + _IS_SYNC = True @@ -122,6 +124,7 @@ def __init__( key_vault_coll: Collection[_DocumentTypeArg], mongocryptd_client: Optional[MongoClient[_DocumentTypeArg]], opts: AutoEncryptionOpts, + proxy_options: Optional[_ProxyOptions] = None, ): """Internal class to perform I/O on behalf of pymongocrypt.""" self.client_ref: Any @@ -139,6 +142,7 @@ def __init__( ), ) self.mongocryptd_client = mongocryptd_client + self._proxy_options = proxy_options self.opts = opts self._spawned = False self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC) @@ -181,6 +185,7 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: connect_timeout=connect_timeout, socket_timeout=connect_timeout, ssl_context=ctx, + proxy_options=self._proxy_options, ) try: conn = _connect_kms( @@ -357,6 +362,9 @@ def __init__(self, client: MongoClient[_DocumentTypeArg], opts: AutoEncryptionOp :param client: The encrypted MongoClient. :param opts: The encrypted client's :class:`AutoEncryptionOpts`. """ + _check_kms_proxy_options( + client.options.pool_options._proxy_options, opts._kms_connect_callback + ) if opts._schema_map is None: schema_map = None else: @@ -384,6 +392,7 @@ def _get_internal_client( encrypter._internal_client = internal_client return internal_client + # KMS requests use the encrypted client's SOCKS5 proxy options, not the key vault client's. if opts._key_vault_client is not None: key_vault_client = opts._key_vault_client else: @@ -406,6 +415,7 @@ def _get_internal_client( key_vault_coll, # type:ignore[arg-type] mongocryptd_client, opts, + proxy_options=client.options.pool_options._proxy_options, ) self._auto_encrypter = AutoEncrypter( io_callbacks, @@ -569,7 +579,8 @@ def __init__( collection. Data keys are protected with encryption by a KMS provider. :param key_vault_client: A MongoClient connected to a MongoDB cluster - containing the `key_vault_namespace` collection. + containing the `key_vault_namespace` collection. Its SOCKS5 proxy + options are also used for KMS requests. :param codec_options: An instance of :class:`~bson.codec_options.CodecOptions` to use when encoding a value for encryption and decoding the decrypted BSON value. This @@ -601,8 +612,11 @@ def __init__( deadline and closes any socket the callback yields later. For an ordinary HTTP proxy, pass :class:`~pymongo.encryption_options.HTTPProxyKMSConnect`. - Defaults to ``None``, meaning the driver connects to KMS hosts - directly. + Defaults to ``None``, meaning the driver uses the SOCKS5 proxy + configured on ``key_vault_client``, or connects directly if none + is configured. Cannot be combined with ``proxyHost`` on + ``key_vault_client``; doing so raises + :class:`~pymongo.errors.ConfigurationError`. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. @@ -632,6 +646,9 @@ def __init__( if not any(cls.__name__ == "MongoClient" for cls in type(key_vault_client).__mro__): raise TypeError(f"MongoClient required but given {type(key_vault_client).__name__}") + _check_kms_proxy_options( + key_vault_client.options.pool_options._proxy_options, kms_connect_callback + ) self._kms_providers = kms_providers self._key_vault_namespace = key_vault_namespace self._key_vault_client = key_vault_client @@ -649,7 +666,11 @@ def __init__( ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( - None, key_vault_coll, None, opts + None, + key_vault_coll, + None, + opts, + proxy_options=key_vault_client.options.pool_options._proxy_options, ) self._encryption = ExplicitEncrypter( self._io_callbacks, diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index d9816b07a2..059d0c72c5 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -99,6 +99,7 @@ UpdateOne, _Op, ) +from pymongo.pool_options import _ProxyOptions from pymongo.read_preferences import ReadPreference, _ServerMode from pymongo.results import ClientBulkWriteResult from pymongo.server_description import ServerDescription @@ -156,6 +157,8 @@ T, ] +_REDACTED_REPR_OPTS = frozenset(["username", "password", "proxyusername", "proxypassword"]) + _IS_SYNC = True _WriteOp = Union[ @@ -338,6 +341,18 @@ def __init__( connecting a new socket to a server before concluding the server is unavailable. ``0`` or ``None`` means no timeout. Defaults to ``20000`` (20 seconds). + - `proxyHost` (string): The hostname or IP address of a SOCKS5 proxy. + IPv6 literals may be enclosed in brackets. Defaults to no proxy. + Routes MongoDB and automatic encryption KMS connections through + the proxy. Cannot be combined with ``kms_connect_callback`` in + ``auto_encryption_opts``; doing so raises + :class:`~pymongo.errors.ConfigurationError`. + - `proxyPort` (integer): The TCP port of the SOCKS5 proxy. Defaults + to ``1080`` when ``proxyHost`` is specified. + - `proxyUsername` (string): The username for SOCKS5 username/password + authentication. Must be specified together with ``proxyPassword``. + - `proxyPassword` (string): The password for SOCKS5 username/password + authentication. Must be specified together with ``proxyUsername``. - `server_selector`: (callable or None) Optional, user-provided function that augments server selection rules. The function should accept as an argument a list of @@ -671,6 +686,10 @@ def validator(host: str) -> bool: .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.20 + Added the ``proxyHost``, ``proxyPort``, ``proxyUsername``, and + ``proxyPassword`` keyword arguments and URI options. + .. versionchanged:: 4.19 Added the ``srv_host_validator`` keyword argument. @@ -878,6 +897,7 @@ def validator(host: str) -> bool: warn=True, normalize=False, srv_max_hosts=srv_max_hosts, + validate_proxy=False, ) is_srv = entity.startswith(SRV_SCHEME) self._seeds.update(res["nodelist"]) @@ -1003,6 +1023,7 @@ def _resolve_srv(self) -> None: srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, srv_host_validator=srv_host_validator, + validate_proxy=False, ) seeds.update(res["nodelist"]) opts = res["options"] @@ -1378,12 +1399,15 @@ def options(self) -> ClientOptions: """ return self._options - def eq_props(self) -> tuple[tuple[_Address, ...], Optional[str], Optional[str], str]: + def eq_props( + self, + ) -> tuple[tuple[_Address, ...], Optional[str], Optional[str], str, Optional[_ProxyOptions]]: return ( tuple(sorted(self._resolve_srv_info["seeds"])), self._options.replica_set_name, self._resolve_srv_info["fqdn"], self._resolve_srv_info["srv_service_name"], + self._options.pool_options._proxy_options, ) def __eq__(self, other: Any) -> bool: @@ -1432,7 +1456,7 @@ def option_repr(option: str, value: Any) -> str: options.extend( option_repr(key, self._options._options[key]) for key in self._options._options - if key not in set(self._constructor_args) and key != "username" and key != "password" + if key not in set(self._constructor_args) and key not in _REDACTED_REPR_OPTS ) return ", ".join(options) diff --git a/pymongo/synchronous/socks5_support.py b/pymongo/synchronous/socks5_support.py new file mode 100644 index 0000000000..9a9662e29b --- /dev/null +++ b/pymongo/synchronous/socks5_support.py @@ -0,0 +1,130 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Synchronous SOCKS5 proxy connection establishment.""" + +from __future__ import annotations + +import socket +from typing import TYPE_CHECKING, Optional + +from pymongo.pool_shared import _configure_tcp_socket, _new_tcp_socket +from pymongo.socks5_shared import ( + _SOCKS5_ATYP_DOMAIN, + _SOCKS5_AUTH_USER_PASS, + _proxy_connect, + _proxy_deadline, + _proxy_getaddrinfo, + _proxy_remaining, + _proxy_socket_timeout, + _socks5_auth_request, + _socks5_bound_address_size, + _socks5_check_auth_reply, + _socks5_check_connect_reply, + _socks5_check_greeting_reply, + _socks5_greeting, + _socks5_recv, + _socks5_request, + _socks5_sendall, + _Socks5Error, + _Socks5Timeout, +) + +if TYPE_CHECKING: + from pymongo.pool_options import PoolOptions, _ProxyOptions + from pymongo.typings import _Address + +_IS_SYNC = True + + +def _socks5_read_exact(sock: socket.socket, size: int, deadline: Optional[float]) -> bytes: + data = bytearray() + while len(data) < size: + chunk = _socks5_recv(sock, size - len(data), deadline) + if not chunk: + raise _Socks5Error("SOCKS5 handshake failed: unexpected EOF") + data.extend(chunk) + return bytes(data) + + +def _socks5_handshake( + sock: socket.socket, address: _Address, proxy: _ProxyOptions, deadline: Optional[float] +) -> None: + greeting, allowed = _socks5_greeting(proxy) + _socks5_sendall(sock, greeting, deadline) + method = _socks5_check_greeting_reply(_socks5_read_exact(sock, 2, deadline), allowed) + if method == _SOCKS5_AUTH_USER_PASS: + _socks5_sendall(sock, _socks5_auth_request(proxy), deadline) + _socks5_check_auth_reply(_socks5_read_exact(sock, 2, deadline)) + _socks5_sendall(sock, _socks5_request(address), deadline) + atyp = _socks5_check_connect_reply(_socks5_read_exact(sock, 4, deadline)) + domain_length = (_socks5_read_exact(sock, 1, deadline))[0] if atyp == _SOCKS5_ATYP_DOMAIN else 0 + bound_size = _socks5_bound_address_size(atyp, domain_length) + _socks5_read_exact(sock, bound_size, deadline) + + +def _create_proxy_connection( + address: _Address, options: PoolOptions, proxy: _ProxyOptions +) -> socket.socket: + deadline = _proxy_deadline(options) + family = socket.AF_INET + if socket.has_ipv6 and proxy.host != "localhost": + family = socket.AF_UNSPEC + try: + results = _proxy_getaddrinfo(proxy, family, deadline) + except TimeoutError as exc: + raise _Socks5Timeout("SOCKS5 proxy timed out") from exc + except OSError as exc: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {exc}") from exc + + err: Optional[OSError] = None + for af, socktype, proto, _, sockaddr in results: + sock = _new_tcp_socket(af, socktype, proto) + connected = False + try: + _configure_tcp_socket(sock, non_blocking=not _IS_SYNC) + _proxy_connect(sock, sockaddr, deadline) + connected = True + _socks5_handshake(sock, address, proxy, deadline) + sock.settimeout(_proxy_socket_timeout(options)) + return sock + except TimeoutError as exc: + sock.close() + if connected: + raise _Socks5Timeout("SOCKS5 proxy timed out") from exc + try: + _proxy_remaining(deadline) + except _Socks5Timeout as timeout: + raise timeout from exc + err = _Socks5Timeout("SOCKS5 proxy timed out") + err.__cause__ = exc + except _Socks5Error: + sock.close() + raise + except OSError as exc: + sock.close() + err = exc + # A connected proxy's handshake failure must not try a different + # resolved proxy address; the failure is specific to the endpoint. + if connected: + raise _Socks5Error(f"SOCKS5 handshake failed: {exc}") from exc + except BaseException: + sock.close() + raise + + if isinstance(err, socket.timeout): + raise err + if err is not None: + raise _Socks5Error(f"SOCKS5 proxy connection failed: {err}") from err + raise _Socks5Error("SOCKS5 proxy connection failed: getaddrinfo returned no addresses") diff --git a/pymongo/synchronous/uri_parser.py b/pymongo/synchronous/uri_parser.py index 9a7fcf0fae..fdbbde989c 100644 --- a/pymongo/synchronous/uri_parser.py +++ b/pymongo/synchronous/uri_parser.py @@ -137,6 +137,7 @@ def _parse_srv( srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, srv_host_validator: Optional[Callable[[str], bool]] = None, + validate_proxy: bool = True, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -199,7 +200,7 @@ def _parse_srv( else: nodes = split_hosts(hosts, default_port=default_port) - _check_options(nodes, options) + _check_options(nodes, options, validate_proxy=validate_proxy) return { "nodelist": nodes, diff --git a/pymongo/topology_shared.py b/pymongo/topology_shared.py index 23e47557db..70ba88c248 100644 --- a/pymongo/topology_shared.py +++ b/pymongo/topology_shared.py @@ -262,6 +262,7 @@ def _create_pool_for_monitor(self, address: _Address) -> _PoolT: driver=options.driver, pause_enabled=False, server_api=options.server_api, + proxy_options=options._proxy_options, ) return self._settings.pool_class( diff --git a/pymongo/uri_parser_shared.py b/pymongo/uri_parser_shared.py index 5e34c2d702..211f04b9d5 100644 --- a/pymongo/uri_parser_shared.py +++ b/pymongo/uri_parser_shared.py @@ -23,7 +23,7 @@ import re import sys import warnings -from collections.abc import Mapping, MutableMapping, Sized +from collections.abc import Collection, Iterable, Mapping, MutableMapping from typing import ( TYPE_CHECKING, Any, @@ -35,6 +35,7 @@ from pymongo.client_options import _parse_ssl_options from pymongo.common import ( + _URI_OPTS_MUST_RAISE, INTERNAL_URI_OPTION_NAME_MAP, URI_OPTIONS_DEPRECATION_MAP, _CaseInsensitiveDictionary, @@ -249,6 +250,8 @@ def _parse_options(opts: str, delim: Optional[str]) -> _CaseInsensitiveDictionar options.setdefault(key, []).append(value) else: if key in options: + if key.lower() in _URI_OPTS_MUST_RAISE: + raise InvalidURI(f"Duplicate URI option '{key}' is not allowed") warnings.warn(f"Duplicate URI option '{key}'.", stacklevel=2) if key.lower() == "authmechanismproperties": val = value @@ -365,6 +368,10 @@ def _normalize_options(options: _CaseInsensitiveDictionary) -> _CaseInsensitiveD # Implicit options are logically the same as tlsInsecure. options[opt] = tlsinsecure + for optname in ("proxyusername", "proxypassword"): + if options.get(optname) == "": + options.pop(optname) + for optname in list(options): intname = INTERNAL_URI_OPTION_NAME_MAP.get(optname, None) if intname is not None: @@ -479,7 +486,27 @@ def split_hosts(hosts: str, default_port: Optional[int] = DEFAULT_PORT) -> list[ ) -def _check_options(nodes: Sized, options: Mapping[str, Any]) -> None: +def _check_proxy_options(nodes: Iterable[_Address], options: Mapping[str, Any]) -> None: + proxy_host = options.get("proxyhost") + proxy_port = options.get("proxyport") + proxy_username = options.get("proxyusername") + proxy_password = options.get("proxypassword") + if proxy_host is None: + if proxy_port is not None or proxy_username or proxy_password: + raise ConfigurationError( + "Cannot specify proxyPort, proxyUsername, or proxyPassword without proxyHost" + ) + return + + if bool(proxy_username) != bool(proxy_password): + raise ConfigurationError("Both proxyUsername and proxyPassword must be specified together") + if any(host.endswith(".sock") for host, _ in nodes): + raise ConfigurationError("Cannot specify proxyHost with Unix domain sockets") + + +def _check_options( + nodes: Collection[_Address], options: Mapping[str, Any], validate_proxy: bool = True +) -> None: # Ensure directConnection was not True if there are multiple seeds. if len(nodes) > 1 and options.get("directconnection"): raise ConfigurationError("Cannot specify multiple hosts with directConnection=true") @@ -492,6 +519,9 @@ def _check_options(nodes: Sized, options: Mapping[str, Any]) -> None: if options.get("replicaset"): raise ConfigurationError("Cannot specify replicaSet with loadBalanced=true") + if validate_proxy: + _check_proxy_options(nodes, options) + def _parse_kms_tls_options( kms_tls_options: Optional[Mapping[str, Any]], @@ -536,6 +566,7 @@ def _validate_uri( warn: bool = False, normalize: bool = True, srv_max_hosts: Optional[int] = None, + validate_proxy: bool = True, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -615,7 +646,7 @@ def _validate_uri( else: nodes = split_hosts(hosts, default_port=default_port) - _check_options(nodes, options) + _check_options(nodes, options, validate_proxy=validate_proxy) return { "nodelist": nodes, diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index f5e79b5048..4b06a1c1d3 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -40,6 +40,7 @@ from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from types import SimpleNamespace from typing import Any from unittest import mock @@ -55,9 +56,9 @@ KMSConnectContext, ) from pymongo.errors import ConfigurationError, ConnectionFailure, EncryptionError, NetworkTimeout -from pymongo.pool_options import PoolOptions +from pymongo.pool_options import PoolOptions, _ProxyOptions from pymongo.ssl_support import get_ssl_context -from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM +from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM, LOCAL_MASTER_KEY pytestmark = [pytest.mark.encryption, pytest.mark.asyncio] @@ -167,15 +168,15 @@ def client_encryption(self, kms_providers, key_vault_namespace, client, kms_conn kms_connect_callback=kms_connect_callback, ) - def simple_client(self): + def simple_client(self, *args, **kwargs): """A lazily-connecting client for this API.""" if self.is_async: from pymongo.asynchronous.mongo_client import AsyncMongoClient - return AsyncMongoClient() + return AsyncMongoClient(*args, **kwargs) from pymongo import MongoClient - return MongoClient() + return MongoClient(*args, **kwargs) ASYNC = Facade(is_async=True) @@ -1058,6 +1059,86 @@ async def test_datagram_socket_from_callback_is_rejected(api): await api.connect(_KMS_ADDRESS, _pool_options(), api.callback_returning(left), 10.0) +@both_apis +@pytest.mark.skipif(not _HAVE_PYMONGOCRYPT, reason="pymongocrypt is not installed") +async def test_socks5_proxy_options_flow_to_kms(api): + module = api.encryption() + proxy = _ProxyOptions("10.0.0.2") + client = api.simple_client("mongodb://localhost/?proxyHost=10.0.0.2", connect=False) + clients = [] + explicit = None + monkeypatch = pytest.MonkeyPatch() + try: + opts = mock.Mock( + _kms_connect_callback=None, + _kms_ssl_contexts=mock.Mock(return_value={"aws": object()}), + ) + io = module._EncryptionIO(client, client.keyvault.datakeys, None, opts, proxy_options=proxy) + connection = mock.MagicMock() + + def connect(_address, options, _callback, _timeout): + assert options._proxy_options == proxy + return connection + + patched_connect = ( + mock.AsyncMock(side_effect=connect) if api.is_async else mock.Mock(side_effect=connect) + ) + monkeypatch.setattr(module, "_connect_kms", patched_connect) + if api.is_async: + monkeypatch.setattr(module, "async_socket_sendall", mock.AsyncMock()) + request = SimpleNamespace( + endpoint="kms.example:443", + message=b"request", + kms_provider="aws", + usleep=0, + bytes_needed=0, + ) + await api.maybe_await(io.kms_request(request)) + connection.close.assert_called_once_with() + + for bypass in (False, True): + encrypted = api.simple_client( + "mongodb://localhost/?proxyHost=10.0.0.1", + auto_encryption_opts=AutoEncryptionOpts( + {"local": {"key": LOCAL_MASTER_KEY}}, + "keyvault.datakeys", + key_vault_client=client, + bypass_auto_encryption=bypass, + ), + connect=False, + ) + clients.append(encrypted) + callbacks = encrypted._encrypter._auto_encrypter.callback + assert callbacks._proxy_options == _ProxyOptions("10.0.0.1") + assert (callbacks.client_ref is None) is bypass + assert callbacks.mongocryptd_client.options.pool_options._proxy_options is None + + explicit = api.client_encryption( + {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", client, None + ) + assert explicit._io_callbacks._proxy_options == proxy + callback = api.callback(lambda _context: None) + opts = AutoEncryptionOpts({}, "keyvault.datakeys", kms_connect_callback=callback) + client_with_proxy = mock.Mock( + options=mock.Mock(pool_options=PoolOptions(proxy_options=proxy)) + ) + for create in ( + lambda: module._Encrypter(client_with_proxy, opts), + lambda: api.client_encryption( + {"local": {"key": LOCAL_MASTER_KEY}}, "keyvault.datakeys", client, callback + ), + ): + with pytest.raises(ConfigurationError, match="kms_connect_callback with proxyHost"): + create() + finally: + monkeypatch.undo() + if explicit is not None: + await api.maybe_await(explicit.close()) + for encrypted in clients: + await api.maybe_await(encrypted.close()) + await api.maybe_await(client.close()) + + @both_apis async def test_kms_request_does_not_retry_a_contract_violation(api): # The no-retry guarantee lives in kms_request, not _connect_kms. diff --git a/test/test_socks5.py b/test/test_socks5.py new file mode 100644 index 0000000000..4d93046409 --- /dev/null +++ b/test/test_socks5.py @@ -0,0 +1,960 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for SOCKS5 proxy connection establishment.""" + +from __future__ import annotations + +import asyncio +import os +import select +import socket +import ssl +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any, NamedTuple, Optional, cast +from unittest import mock +from urllib.parse import quote_plus + +import pytest + +sys.path[0:0] = [""] + +from pymongo import monitoring +from pymongo.hello import HelloCompat +from pymongo.pool_options import PoolOptions, _ProxyOptions +from pymongo.pool_shared import ( + _async_configured_socket, + _async_create_connection, + _configured_socket, + _create_connection, +) +from pymongo.socks5_shared import ( + _SOCKS5_ATYP_DOMAIN, + _SOCKS5_ATYP_IPV4, + _SOCKS5_ATYP_IPV6, + _SOCKS5_AUTH_NO_ACCEPTABLE, + _SOCKS5_AUTH_NO_AUTH, + _SOCKS5_AUTH_SUCCESS, + _SOCKS5_AUTH_USER_PASS, + _SOCKS5_AUTH_VERSION, + _SOCKS5_CMD_CONNECT, + _SOCKS5_REPLY_SUCCESS, + _SOCKS5_RESERVED, + _SOCKS5_VERSION, + _Socks5Error, + _Socks5Timeout, +) +from test import IntegrationTest, PyMongoTestCase, client_context +from test.asynchronous import AsyncIntegrationTest, AsyncPyMongoTestCase, async_client_context +from test.asynchronous.utils import async_wait_until +from test.helpers_shared import CERT_PATH, TLS_OPTIONS + +pytestmark = pytest.mark.default + +_SOCKS5_AUTH_GSSAPI_METHOD = 0x01 +_SOCKS5_AUTH_FAILURE_STATUS = 0x01 +_SOCKS4_VERSION = 0x04 + + +class _ProseCase(NamedTuple): + mapped: bool + proxy_port: int + credentials: Optional[tuple[str, str]] + succeeds: bool + + +class TestSocks5(PyMongoTestCase): + def _create_server(self, **kwargs: Any) -> _Socks5Server: + server = _Socks5Server(**kwargs) + self.addCleanup(server.close) + return server + + def _connect( + self, + server: _Socks5Server, + address: tuple[str, int] = ("localhost", 27017), + username: Optional[str] = None, + password: Optional[str] = None, + timeout: float = 2, + ) -> socket.socket: + sock = _create_connection( + address, + PoolOptions( + connect_timeout=timeout, + proxy_options=_ProxyOptions( + server.address[0], server.address[1], username, password + ), + ), + ) + self.addCleanup(sock.close) + return sock + + def test_greeting_and_auth(self) -> None: + server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) + self._connect(server, ("mongodb.example", 27017), "üser", "påss").close() + server.close() + self.assertEqual( + server.greeting, + bytes((_SOCKS5_VERSION, 2, _SOCKS5_AUTH_NO_AUTH, _SOCKS5_AUTH_USER_PASS)), + ) + self.assertEqual( + server.auth, + bytes((_SOCKS5_AUTH_VERSION, len("üser".encode()))) + + "üser".encode() + + bytes((len("påss".encode()),)) + + "påss".encode(), + ) + self.assertEqual( + server.request, + bytes( + ( + _SOCKS5_VERSION, + _SOCKS5_CMD_CONNECT, + _SOCKS5_RESERVED, + _SOCKS5_ATYP_DOMAIN, + len(b"mongodb.example"), + ) + ) + + b"mongodb.example" + + (27017).to_bytes(2, "big"), + ) + + server = self._create_server(method=_SOCKS5_AUTH_NO_AUTH) + self._connect(server, ("127.0.0.1", 27017), "user", "password").close() + server.close() + self.assertEqual(server.auth, b"") + + for method in ( + _SOCKS5_AUTH_USER_PASS, + _SOCKS5_AUTH_GSSAPI_METHOD, + _SOCKS5_AUTH_NO_ACCEPTABLE, + ): + server = self._create_server(method=method) + with self.subTest(method=method), self.assertRaises(_Socks5Error): + self._connect(server) + server.close() + self.assertEqual(server.greeting, bytes((_SOCKS5_VERSION, 1, _SOCKS5_AUTH_NO_AUTH))) + + server = self._create_server( + method=_SOCKS5_AUTH_USER_PASS, auth_status=_SOCKS5_AUTH_FAILURE_STATUS + ) + with self.assertRaises(_Socks5Error): + self._connect(server, username="user", password="password") + server.close() + + def test_connect_request_and_bound_replies(self) -> None: + for host, expected in ( + ("192.0.2.1", bytes((_SOCKS5_ATYP_IPV4,)) + socket.inet_aton("192.0.2.1")), + ( + "2001:db8::1", + bytes((_SOCKS5_ATYP_IPV6,)) + socket.inet_pton(socket.AF_INET6, "2001:db8::1"), + ), + ( + "münich.example", + bytes((_SOCKS5_ATYP_DOMAIN, len("xn--mnich-kva.example"))) + + b"xn--mnich-kva.example", + ), + ): + server = self._create_server() + self._connect(server, (host, 27017)).close() + server.close() + self.assertEqual( + server.request, + bytes((_SOCKS5_VERSION, _SOCKS5_CMD_CONNECT, _SOCKS5_RESERVED)) + + expected + + (27017).to_bytes(2, "big"), + ) + + for atyp in (_SOCKS5_ATYP_IPV4, _SOCKS5_ATYP_DOMAIN, _SOCKS5_ATYP_IPV6): + server = self._create_server(bound_atyp=atyp, payload=b"mongo") + sock = self._connect(server) + self.assertEqual(sock.recv(5), b"mongo") + sock.close() + server.close() + + for reply in range(1, 9): + server = self._create_server(reply=reply) + with self.subTest(reply=reply), self.assertRaises(_Socks5Error): + self._connect(server) + server.close() + + def test_tcp_socket_without_cloexec_falls_back(self) -> None: + from pymongo import pool_shared + + sock = socket.socket() + self.addCleanup(sock.close) + with mock.patch.object( + pool_shared.socket, "socket", side_effect=[OSError("unsupported"), sock] + ): + self.assertIs(pool_shared._new_tcp_socket(socket.AF_INET, socket.SOCK_STREAM, 0), sock) + + def test_client_proxy_options_and_repr(self) -> None: + from pymongo.synchronous.mongo_client import MongoClient + from pymongo.synchronous.topology import Topology + + c1 = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) + c2 = MongoClient("mongodb://localhost/?proxyHost=127.0.0.1", connect=False) + c3 = MongoClient( + "mongodb://localhost/?proxyHost=localhost&proxyPort=1234" + "&proxyUsername=uri-user&proxyPassword=uri-password", + proxyHost="[::1]", + proxyPort=5678, + proxyUsername="", + proxyPassword="", + connect=False, + ) + c4 = MongoClient( + "mongodb://user:pass@localhost/?proxyHost=127.0.0.1" + "&proxyUsername=proxyuser&proxyPassword=hunter2", + connect=False, + ) + c5 = MongoClient( + "mongodb://localhost/?proxyPort=1081", proxyHost="proxy.example", connect=False + ) + c6 = MongoClient( + "mongodb://localhost/?proxyHost=proxy.example&proxyUsername=u", + proxyPassword="p", + connect=False, + ) + for client in (c1, c2, c3, c4, c5, c6): + self.addCleanup(client.close) + + topology = Topology(c1._topology_settings) + expected = _ProxyOptions("127.0.0.1") + self.assertEqual( + topology._create_pool_for_server(("localhost", 27017)).opts._proxy_options, expected + ) + self.assertEqual( + topology._create_pool_for_monitor(("localhost", 27017)).opts._proxy_options, expected + ) + self.assertEqual(c1, c2) + self.assertNotEqual(c1, c3) + self.assertEqual(c3.options.pool_options._proxy_options, _ProxyOptions("::1", 5678)) + self.assertEqual( + c5.options.pool_options._proxy_options, _ProxyOptions("proxy.example", 1081) + ) + self.assertEqual( + c6.options.pool_options._proxy_options, _ProxyOptions("proxy.example", 1080, "u", "p") + ) + for secret in ("hunter2", "proxyuser", "pass"): + self.assertNotIn(secret, repr(c4)) + self.assertNotIn("hunter2", repr(c4.options.pool_options._proxy_options)) + + def test_bad_versions_eof_timeout_and_tls_sni(self) -> None: + for server, creds in ( + (self._create_server(method_version=_SOCKS4_VERSION), (None, None)), + (self._create_server(reply_version=_SOCKS4_VERSION), (None, None)), + (self._create_server(eof_stage="greeting"), (None, None)), + (self._create_server(method=_SOCKS5_AUTH_USER_PASS, eof_stage="auth"), ("u", "p")), + (self._create_server(eof_stage="connect"), (None, None)), + ): + with self.assertRaises(_Socks5Error): + self._connect(server, username=creds[0], password=creds[1]) + server.close() + + server = self._create_server(stall=True) + with self.assertRaisesRegex(socket.timeout, "SOCKS5 proxy timed out"): + self._connect(server, timeout=0.1) + server.close() + + sni_names: list[Optional[str]] = [] + tls_server = _TLSServer(sni_names) + self.addCleanup(tls_server.close) + proxy = self._create_server(target=tls_server.address) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + sock = _configured_socket( + ("mongodb.example", tls_server.address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, ctx), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), + ), + ) + sock.close() + self.assertEqual(sni_names, ["mongodb.example"]) + + def test_socks5_errors_are_not_labeled_system_overloaded(self) -> None: + from pymongo.errors import AutoReconnect, NetworkTimeout + from pymongo.pool_shared import _raise_connection_failure + from pymongo.synchronous.pool import Pool + + pool = Pool(("localhost", 27017), PoolOptions(connect_timeout=1)) + self.addCleanup(pool.close) + for cause, overloaded in ( + (_Socks5Error("boom"), False), + (_Socks5Timeout("SOCKS5 proxy timed out"), False), + (OSError("boom"), True), + ): + with self.assertRaises((AutoReconnect, NetworkTimeout)) as raised: + _raise_connection_failure(("localhost", 27017), cause) + pool._handle_connection_error(raised.exception) + self.assertEqual(raised.exception.has_error_label("SystemOverloadedError"), overloaded) + self.assertEqual(raised.exception.has_error_label("RetryableError"), overloaded) + + def test_sync_proxy_error_and_fallback_paths(self) -> None: + import pymongo.synchronous.socks5_support as socks5_module + + options = PoolOptions(connect_timeout=1, proxy_options=_ProxyOptions("proxy.invalid", 1080)) + target = "pymongo.socks5_shared.socket.getaddrinfo" + for err, exc_type in ( + (socket.gaierror("failed"), _Socks5Error), + (socket.timeout("timed out"), _Socks5Timeout), + ): + with mock.patch(target, side_effect=err), self.assertRaises(exc_type): + _create_connection(("localhost", 27017), options) + + with ( + mock.patch(target, return_value=[]), + self.assertRaisesRegex(_Socks5Error, "no addresses"), + ): + _create_connection(("localhost", 27017), options) + + dead = (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", 1)) + for err, exc_type in ( + (ConnectionRefusedError("refused"), _Socks5Error), + (socket.timeout(), _Socks5Timeout), + ): + with ( + mock.patch(target, return_value=[dead]), + mock.patch.object(socks5_module, "_proxy_connect", side_effect=err), + self.assertRaises(exc_type), + ): + _create_connection(("localhost", 27017), options) + + server = self._create_server() + live = (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address) + orig_connect = socks5_module._proxy_connect + + def fail_first(sock: socket.socket, addr: Any, deadline: Optional[float]) -> None: + if addr == ("127.0.0.1", 1): + raise ConnectionRefusedError("refused") + orig_connect(sock, addr, deadline) + + with ( + mock.patch(target, return_value=[dead, live]), + mock.patch.object(socks5_module, "_proxy_connect", side_effect=fail_first), + ): + self._connect(server).close() + server.close() + + with ( + mock.patch.object(socks5_module, "_proxy_deadline", return_value=0), + mock.patch.object(socks5_module, "_proxy_getaddrinfo", return_value=[dead, dead]), + mock.patch.object( + socks5_module, "_proxy_connect", side_effect=socket.timeout("timed out") + ) as connect, + self.assertRaises(_Socks5Timeout), + ): + _create_connection(("mongodb.example", 27017), options) + connect.assert_called_once() + + for error in (OSError("handshake failed"), KeyboardInterrupt): + server = self._create_server() + opts = PoolOptions(connect_timeout=1, proxy_options=_ProxyOptions(*server.address)) + with ( + mock.patch.object(socks5_module, "_socks5_sendall", side_effect=error), + self.assertRaises( + _Socks5Error if isinstance(error, OSError) else KeyboardInterrupt + ), + ): + _create_connection(("localhost", 27017), opts) + server.close() + + +class _CommandListener(monitoring.CommandListener): + def __init__(self) -> None: + self.connection_ids: list[tuple[str, Optional[int]]] = [] + + def started(self, event: monitoring.CommandStartedEvent) -> None: + self.connection_ids.append(event.connection_id) + + def succeeded(self, event: monitoring.CommandSucceededEvent) -> None: + self.connection_ids.append(event.connection_id) + + def failed(self, event: monitoring.CommandFailedEvent) -> None: + self.connection_ids.append(event.connection_id) + + +def _start_socks5_proxies(nodes: list[tuple[str, int]]) -> list[subprocess.Popen[bytes]]: + tools = Path(os.environ.get("DRIVERS_TOOLS", "drivers-evergreen-tools")) + script = tools / ".evergreen" / "socks5srv.py" + host, port = nodes[0] + seed = f"[{host}]:{port}" if ":" in host else f"{host}:{port}" + mapping = f"localhost:12345 to {seed}" + processes = [] + try: + for proxy_port, auth in ((1080, True), (1081, False)): + args = [sys.executable, str(script), "--port", str(proxy_port), "--map", mapping] + if auth: + args.extend(("--auth", "username:p4ssw0rd")) + process = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + processes.append(process) + deadline = time.monotonic() + 10 + while True: + if process.poll() is not None: + raise RuntimeError(f"SOCKS5 proxy on port {proxy_port} exited") + try: + with socket.create_connection(("127.0.0.1", proxy_port), timeout=0.1): + break + except OSError: + if time.monotonic() >= deadline: + raise RuntimeError(f"SOCKS5 proxy on port {proxy_port} did not start") + time.sleep(0.1) + return processes + except BaseException: + _stop_socks5_proxies(processes) + raise + + +def _stop_socks5_proxies(processes: list[subprocess.Popen[bytes]]) -> None: + for process in processes: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +async def _maybe_await(value: Any) -> Any: + return await value if hasattr(value, "__await__") else value + + +def _prose_hosts(nodes: list[tuple[str, int]], mapped: bool) -> str: + if mapped: + return "localhost:12345" + return ",".join( + f"[{host}]:{port}" if ":" in host and not host.startswith("[") else f"{host}:{port}" + for host, port in nodes + ) + + +async def _run_prose_case( + test: Any, + async_api: bool, + hosts: str, + port: int, + credentials: Optional[tuple[str, str]], + succeeds: bool, + use_uri: bool, + listener: Optional[_CommandListener] = None, +) -> None: + from pymongo.errors import ConnectionFailure, OperationFailure + + if async_api: + from pymongo.asynchronous.mongo_client import AsyncMongoClient as Client + else: + from pymongo.synchronous.mongo_client import MongoClient as Client + proxy = {"proxyHost": "localhost", "proxyPort": port} + if credentials: + proxy.update(proxyUsername=credentials[0], proxyPassword=credentials[1]) + context = async_client_context if async_api else client_context + options = dict(TLS_OPTIONS) if context.tls else {} + options.update(test.credentials) + options["serverSelectionTimeoutMS"] = 2500 + if hosts == "localhost:12345": + options["directConnection"] = True + if listener: + options["event_listeners"] = [listener] + if use_uri: + query = "&".join(f"{key}={quote_plus(str(value))}" for key, value in proxy.items()) + hosts = f"mongodb://{hosts}/?{query}" + else: + options.update(proxy) + client = Client(hosts, **options) + try: + try: + await _maybe_await(client.admin.command(HelloCompat.CMD)) + except ConnectionFailure: + if succeeds: + raise + return + if not succeeds: + raise AssertionError("Expected the SOCKS5 connection to fail") + if listener: + with pytest.raises(OperationFailure): + await _maybe_await(client.admin.command("nonexistent_socks5_test_command")) + finally: + await _maybe_await(client.close()) + + +async def _test_prose_table(test: Any, async_api: bool) -> None: + cases = ( + _ProseCase(True, 1080, None, False), + _ProseCase(True, 1081, None, True), + _ProseCase(False, 1080, None, False), + _ProseCase(False, 1081, None, True), + _ProseCase(True, 1080, ("nonexistentuser", "badauth"), False), + _ProseCase(True, 1081, ("nonexistentuser", "badauth"), True), + _ProseCase(False, 1081, ("nonexistentuser", "badauth"), True), + _ProseCase(True, 1080, ("username", "p4ssw0rd"), True), + _ProseCase(False, 1080, ("username", "p4ssw0rd"), True), + ) + for case in cases: + for use_uri in (True, False): + await _run_prose_case( + test, + async_api, + _prose_hosts(test.nodes, case.mapped), + case.proxy_port, + case.credentials, + case.succeeds, + use_uri, + ) + + +async def _test_command_monitoring(test: Any, async_api: bool) -> None: + proxy_addresses = {(host, port) for host in ("localhost", "127.0.0.1") for port in (1080, 1081)} + for mapped in (True, False): + listener = _CommandListener() + await _run_prose_case( + test, async_api, _prose_hosts(test.nodes, mapped), 1081, None, True, False, listener + ) + assert len(listener.connection_ids) >= 4 + for address in listener.connection_ids: + assert address not in proxy_addresses + assert address == (("localhost", 12345) if mapped else address) + if not mapped: + assert address in test.nodes + + +class TestSocks5Prose(IntegrationTest): + @client_context.require_replica_set + def setUp(self) -> None: + super().setUp() + if not os.environ.get("TEST_SOCKS5"): + self.skipTest("SOCKS5 proxies are started only by the SOCKS5 task") + self.nodes = sorted(client_context.nodes) + self.addCleanup(_stop_socks5_proxies, _start_socks5_proxies(self.nodes)) + + def test_connection_string_prose_table(self) -> None: + asyncio.run(_test_prose_table(self, False)) + + def test_command_monitoring_does_not_reference_proxy(self) -> None: + asyncio.run(_test_command_monitoring(self, False)) + + +class _TLSServer: + def __init__(self, sni_names: list[Optional[str]]) -> None: + self.sni_names = sni_names + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.listener.settimeout(3) + self.address = self.listener.getsockname() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + def _serve(self) -> None: + try: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(f"{CERT_PATH}/server.pem") + context.set_servername_callback(lambda _sock, name, _ctx: self.sni_names.append(name)) + with self.listener.accept()[0] as raw: + with context.wrap_socket(raw, server_side=True) as conn: + conn.recv(1) + except OSError: + pass + finally: + self.listener.close() + + def close(self) -> None: + self.listener.close() + self.thread.join(timeout=3) + + +class _Socks5Server: + def __init__( + self, + method: int = _SOCKS5_AUTH_NO_AUTH, + auth_status: int = _SOCKS5_AUTH_SUCCESS, + reply: int = _SOCKS5_REPLY_SUCCESS, + bound_atyp: int = _SOCKS5_ATYP_IPV4, + payload: bytes = b"", + method_version: int = _SOCKS5_VERSION, + reply_version: int = _SOCKS5_VERSION, + eof_stage: Optional[str] = None, + stall: bool = False, + target: Optional[tuple[str, int]] = None, + ) -> None: + self.method = method + self.auth_status = auth_status + self.reply = reply + self.bound_atyp = bound_atyp + self.payload = payload + self.method_version = method_version + self.reply_version = reply_version + self.eof_stage = eof_stage + self.stall = stall + self.target = target + self.started = threading.Event() + self.greeting = b"" + self.auth = b"" + self.request = b"" + self.error: Optional[BaseException] = None + self.listener = socket.socket() + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.listener.settimeout(3) + self.address = self.listener.getsockname() + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + @staticmethod + def _read(sock: socket.socket, size: int) -> bytes: + data = bytearray() + while len(data) < size: + part = sock.recv(size - len(data)) + if not part: + raise EOFError("client closed connection") + data.extend(part) + return bytes(data) + + def _serve(self) -> None: + upstream: Optional[socket.socket] = None + try: + with self.listener.accept()[0] as conn: + conn.settimeout(3) + version, nmethods = self._read(conn, 2) + self.greeting = bytes((version, nmethods)) + self._read(conn, nmethods) + self.started.set() + if self.stall: + conn.recv(1) + return + if self.eof_stage == "greeting": + conn.sendall(bytes((_SOCKS5_VERSION,))) + return + conn.sendall(bytes((self.method_version, self.method))) + if self.method == _SOCKS5_AUTH_USER_PASS: + auth_version, username_length = self._read(conn, 2) + username = self._read(conn, username_length) + password_length = self._read(conn, 1)[0] + password = self._read(conn, password_length) + self.auth = ( + bytes((auth_version, username_length)) + + username + + bytes((password_length,)) + + password + ) + if self.eof_stage == "auth": + conn.sendall(bytes((_SOCKS5_AUTH_VERSION,))) + return + conn.sendall(bytes((_SOCKS5_AUTH_VERSION, self.auth_status))) + if self.auth_status: + return + version, command, reserved, atyp = self._read(conn, 4) + if atyp == _SOCKS5_ATYP_IPV4: + addr = self._read(conn, 4) + elif atyp == _SOCKS5_ATYP_IPV6: + addr = self._read(conn, 16) + else: + length = self._read(conn, 1) + addr = length + self._read(conn, length[0]) + self.request = ( + bytes((version, command, reserved, atyp)) + addr + self._read(conn, 2) + ) + if self.eof_stage == "connect": + conn.sendall(bytes((_SOCKS5_VERSION, _SOCKS5_REPLY_SUCCESS))) + return + if self.target is not None: + upstream = socket.create_connection(self.target, timeout=3) + if self.bound_atyp == _SOCKS5_ATYP_IPV4: + bound = bytes((_SOCKS5_ATYP_IPV4,)) + socket.inet_aton("127.0.0.1") + bytes(2) + elif self.bound_atyp == _SOCKS5_ATYP_IPV6: + bound = ( + bytes((_SOCKS5_ATYP_IPV6,)) + + socket.inet_pton(socket.AF_INET6, "::1") + + bytes(2) + ) + else: + host = b"x" + bound = bytes((_SOCKS5_ATYP_DOMAIN, len(host))) + host + bytes(2) + conn.sendall( + bytes((self.reply_version, self.reply, _SOCKS5_RESERVED)) + bound + self.payload + ) + if self.payload: + return + if upstream is not None: + conn.setblocking(False) + upstream.setblocking(False) + while True: + readable, _, _ = select.select([conn, upstream], [], [], 5) + if not readable: + return + for source in readable: + data = source.recv(65536) + if not data: + return + destination = upstream if source is conn else conn + destination.sendall(data) + else: + while conn.recv(1024): + pass + except BaseException as exc: + self.error = exc + finally: + if upstream is not None: + upstream.close() + self.listener.close() + + def close(self) -> None: + self.listener.close() + self.thread.join(timeout=3) + + +@pytest.mark.default_async +class TestAsyncSocks5(AsyncPyMongoTestCase): + """Exercise the asynchronous proxy paths without duplicating sync scenarios.""" + + def _create_server(self, **kwargs: Any) -> _Socks5Server: + server = _Socks5Server(**kwargs) + self.addAsyncCleanup(server.close) + return server + + async def _connect( + self, + server: _Socks5Server, + address: tuple[str, int] = ("localhost", 27017), + username: Optional[str] = None, + password: Optional[str] = None, + timeout: float = 2, + ) -> socket.socket: + from pymongo.pool_shared import _async_create_connection + + sock = await _async_create_connection( + address, + PoolOptions( + connect_timeout=timeout, + proxy_options=_ProxyOptions( + server.address[0], server.address[1], username, password + ), + ), + ) + self.addAsyncCleanup(sock.close) + return sock + + async def test_protocol_and_connection_errors(self) -> None: + for host, atyp in ( + ("127.0.0.1", _SOCKS5_ATYP_IPV4), + ("2001:db8::1", _SOCKS5_ATYP_IPV6), + ("münich.example", _SOCKS5_ATYP_DOMAIN), + ): + server = self._create_server() + sock = await self._connect(server, (host, 27017)) + sock.close() + server.close() + self.assertEqual(server.request[3], atyp) + + server = self._create_server(method=_SOCKS5_AUTH_USER_PASS) + sock = await self._connect(server, username="üser", password="påss") + sock.close() + server.close() + self.assertIn("üser".encode(), server.auth) + + for method in (_SOCKS5_AUTH_GSSAPI_METHOD, _SOCKS5_AUTH_NO_ACCEPTABLE): + server = self._create_server(method=method) + with self.assertRaises(_Socks5Error): + await self._connect(server) + server.close() + server = self._create_server( + method=_SOCKS5_AUTH_USER_PASS, auth_status=_SOCKS5_AUTH_FAILURE_STATUS + ) + with self.assertRaises(_Socks5Error): + await self._connect(server, username="u", password="p") + server.close() + for reply in range(1, 9): + server = self._create_server(reply=reply) + with self.assertRaises(_Socks5Error): + await self._connect(server) + server.close() + server = self._create_server(eof_stage="greeting") + with self.assertRaisesRegex(_Socks5Error, "unexpected EOF"): + await self._connect(server) + server.close() + + async def test_async_client_options_and_configured_socket(self) -> None: + from pymongo.asynchronous.mongo_client import AsyncMongoClient + from pymongo.asynchronous.uri_parser import parse_uri + from pymongo.pool_shared import _async_configured_socket + + self.assertEqual( + (await parse_uri("mongodb://localhost/?proxyHost=localhost"))["options"]["proxyHost"], + "localhost", + ) + + server = self._create_server() + client = AsyncMongoClient( + "mongodb://localhost/?proxyHost=localhost&proxyPort=1234", + proxyHost=server.address[0], + proxyPort=server.address[1], + connect=False, + ) + self.assertEqual( + client.options.pool_options._proxy_options, + _ProxyOptions(server.address[0], server.address[1]), + ) + await client.close() + + tls = _TLSServer([]) + self.addAsyncCleanup(tls.close) + proxy = self._create_server(target=tls.address) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + sock = await _async_configured_socket( + ("mongodb.example", tls.address[1]), + PoolOptions( + connect_timeout=2, + ssl_context=cast(Any, context), + proxy_options=_ProxyOptions(proxy.address[0], proxy.address[1]), + ), + ) + sock.close() + self.assertEqual(tls.sni_names, ["mongodb.example"]) + + async def test_async_proxy_error_paths(self) -> None: + from pymongo.asynchronous import socks5_support + from pymongo.pool_shared import _async_create_connection + + options = PoolOptions(proxy_options=_ProxyOptions("proxy.invalid", 1080)) + for error, expected in ( + (TimeoutError("timed out"), _Socks5Timeout), + (OSError("resolution failed"), _Socks5Error), + ): + with ( + mock.patch.object(socks5_support, "_async_proxy_getaddrinfo", side_effect=error), + self.assertRaises(expected), + ): + await _async_create_connection(("localhost", 27017), options) + + addresses = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", 1))] + + async def fake_addresses(*_args: Any, **_kwargs: Any) -> Any: + return addresses + + for error, expected in ( + (TimeoutError("timed out"), _Socks5Timeout), + (OSError("refused"), _Socks5Error), + ): + with ( + mock.patch.object(socks5_support, "_async_proxy_getaddrinfo", new=fake_addresses), + mock.patch.object(socks5_support, "_async_proxy_connect", side_effect=error), + self.assertRaises(expected), + ): + await _async_create_connection(("localhost", 27017), options) + with ( + mock.patch.object(socks5_support, "_async_proxy_getaddrinfo", new=fake_addresses), + mock.patch.object(socks5_support, "_async_proxy_connect", side_effect=socket.timeout()), + mock.patch.object( + socks5_support, "_proxy_remaining", side_effect=_Socks5Timeout("expired") + ), + self.assertRaises(_Socks5Timeout), + ): + await _async_create_connection(("localhost", 27017), options) + + async def no_addresses(*_args: Any, **_kwargs: Any) -> Any: + return [] + + with ( + mock.patch.object(socks5_support, "_async_proxy_getaddrinfo", new=no_addresses), + self.assertRaisesRegex(_Socks5Error, "returned no addresses"), + ): + await _async_create_connection(("localhost", 27017), options) + + for error in (OSError("handshake failed"), KeyboardInterrupt): + server = self._create_server() + options = PoolOptions(proxy_options=_ProxyOptions(server.address[0], server.address[1])) + with ( + mock.patch.object(socks5_support, "_async_socks5_sendall", side_effect=error), + self.assertRaisesRegex( + _Socks5Error if isinstance(error, OSError) else KeyboardInterrupt, + "handshake failed" if isinstance(error, OSError) else None, + ), + ): + await _async_create_connection(("localhost", 27017), options) + server.close() + + async def test_async_timeout_cancellation_and_proxy_fallback(self) -> None: + from pymongo.pool_shared import _async_create_connection + + server = self._create_server(stall=True) + with self.assertRaisesRegex(socket.timeout, "SOCKS5 proxy timed out"): + await self._connect(server, timeout=0.1) + server.close() + + server = self._create_server(stall=True) + task = asyncio.create_task(self._connect(server)) + await async_wait_until(server.started.is_set, "receive SOCKS5 greeting", timeout=2) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + server.close() + + dead = ("127.0.0.1", 1) + server = self._create_server() + fake_results = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", dead), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", server.address), + ] + loop = asyncio.get_running_loop() + orig_connect = loop.sock_connect + + async def fail_first(sock: socket.socket, target: tuple[str, int]) -> None: + if target == dead: + raise ConnectionRefusedError("connection failed") + await orig_connect(sock, target) + + with ( + mock.patch( + "pymongo.socks5_shared._getaddrinfo", new=mock.AsyncMock(return_value=fake_results) + ), + mock.patch.object(loop, "sock_connect", new=fail_first), + ): + (await self._connect(server)).close() + server.close() + + with ( + mock.patch("pymongo.socks5_shared._getaddrinfo", side_effect=socket.gaierror("failed")), + self.assertRaises(_Socks5Error), + ): + await _async_create_connection( + ("localhost", 27017), + PoolOptions(proxy_options=_ProxyOptions("proxy.invalid", 1080)), + ) + + +@pytest.mark.default_async +class TestSocks5ProseAsync(AsyncIntegrationTest): + @async_client_context.require_replica_set + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + if not os.environ.get("TEST_SOCKS5"): + self.skipTest("SOCKS5 proxies are started only by the SOCKS5 task") + self.nodes = sorted(async_client_context.nodes) + self.addAsyncCleanup(_stop_socks5_proxies, _start_socks5_proxies(self.nodes)) + + async def test_connection_string_prose_table(self) -> None: + await _test_prose_table(self, True) + + async def test_command_monitoring_does_not_reference_proxy(self) -> None: + await _test_command_monitoring(self, True) diff --git a/test/test_uri_parser.py b/test/test_uri_parser.py index 9ee9ace759..d2c3d4ffe8 100644 --- a/test/test_uri_parser.py +++ b/test/test_uri_parser.py @@ -30,7 +30,19 @@ from bson.binary import JAVA_LEGACY from pymongo import ReadPreference +from pymongo.common import validate_proxy_credential, validate_proxy_host, validate_proxy_port from pymongo.errors import ConfigurationError, InvalidURI +from pymongo.pool_options import PoolOptions +from pymongo.socks5_shared import ( + _proxy_deadline, + _proxy_remaining, + _proxy_socket_timeout, + _socks5_bound_address_size, + _socks5_check_connect_reply, + _socks5_request, + _Socks5Error, + _Socks5Timeout, +) from pymongo.synchronous.uri_parser import parse_uri from pymongo.uri_parser_shared import ( _unquoted_percent, @@ -647,6 +659,77 @@ def test_split_options_duplicate_warning(self): def test_split_options_empty_authsource(self): self.assertRaises(InvalidURI, split_options, "authSource=") + def test_socks5_validation(self): + uri = ( + "mongodb://localhost/?proxyHost=%5B%3A%3A1%5D&proxyPort=invalid&proxyUsername=" + + "é" * 128 + + "&proxyPassword=pass" + ) + self.assertEqual(parse_uri(uri, validate=False)["options"]["proxyPort"], "invalid") + self.assertEqual( + parse_uri("mongodb://localhost/?proxyUsername=&proxyPassword=", normalize=False)[ + "options" + ], + {"proxyUsername": "", "proxyPassword": ""}, + ) + for query in ("proxyHost=", "proxyPort=0", "proxyPort=65536", "proxyUsername=user"): + with ( + self.subTest(query=query), + self.assertRaises((InvalidURI, ValueError, ConfigurationError)), + ): + parse_uri("mongodb://localhost/?" + query) + with self.assertRaises(InvalidURI): + parse_uri("mongodb://localhost/?proxyHost=localhost&proxyhost=a&proxyhost=b") + with self.assertRaises(ConfigurationError): + parse_uri("mongodb://%2Ftmp%2Fmongodb.sock/?proxyHost=localhost") + + def test_socks5_validators(self): + self.assertEqual(validate_proxy_host("proxyHost", "::1"), "::1") + self.assertIsNone(validate_proxy_host("proxyHost", None)) + for host in ("bad/host", "bad host", "[::1", "[invalid]", "localhost:1080"): + with self.subTest(host=host), self.assertRaises(ValueError): + validate_proxy_host("proxyHost", host) + self.assertIsNone(validate_proxy_port("proxyPort", None)) + self.assertEqual(validate_proxy_port("proxyPort", 1080), 1080) + with self.assertRaises(TypeError): + validate_proxy_port("proxyPort", True) + for key in ("proxyUsername", "proxyPassword"): + self.assertEqual(validate_proxy_credential(key, ""), "") + self.assertEqual(validate_proxy_credential(key, "é" * 127 + "a"), "é" * 127 + "a") + with self.assertRaises(ValueError): + validate_proxy_credential(key, "é" * 128) + with self.assertRaises(TypeError): + validate_proxy_credential(key, 1) + + def test_socks5_deadline_helpers(self): + self.assertIsNone(_proxy_deadline(PoolOptions(connect_timeout=None))) + self.assertIsNone(_proxy_remaining(None)) + with self.assertRaises(_Socks5Timeout): + _proxy_remaining(0.0) + with patch("pymongo.socks5_shared._csot.remaining", return_value=0.0): + with self.assertRaises(_Socks5Timeout): + _proxy_deadline(PoolOptions(connect_timeout=1)) + with patch("pymongo.socks5_shared._csot.remaining", return_value=-0.1): + with self.assertRaises(_Socks5Timeout): + _proxy_socket_timeout(PoolOptions(connect_timeout=1)) + + def test_socks5_protocol_validation(self): + self.assertEqual(_socks5_bound_address_size(1), 6) + self.assertEqual(_socks5_bound_address_size(4), 18) + self.assertEqual(_socks5_bound_address_size(3, 5), 7) + for address, message in ( + (("a" * 256, 27017), "invalid domain name length"), + ((".münich.example", 27017), "invalid domain name"), + ): + with self.subTest(address=address), self.assertRaisesRegex(_Socks5Error, message): + _socks5_request(address) + for reply, message in ( + (b"\x05\x00\x01\x01", "invalid reserved byte"), + (b"\x05\x00\x00\x09", "invalid address type"), + ): + with self.subTest(reply=reply), self.assertRaisesRegex(_Socks5Error, message): + _socks5_check_connect_reply(reply) + def test_check_options_conflicts(self): self.assertRaises( ConfigurationError, diff --git a/test/uri_options/proxy-options.json b/test/uri_options/proxy-options.json new file mode 100644 index 0000000000..585546ead7 --- /dev/null +++ b/test/uri_options/proxy-options.json @@ -0,0 +1,139 @@ +{ + "tests": [ + { + "description": "proxyPort without proxyHost", + "uri": "mongodb://localhost/?proxyPort=1080", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "proxyUsername without proxyHost", + "uri": "mongodb://localhost/?proxyUsername=abc", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "proxyPassword without proxyHost", + "uri": "mongodb://localhost/?proxyPassword=def", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "all other proxy options without proxyHost", + "uri": "mongodb://localhost/?proxyPort=1080&proxyUsername=abc&proxyPassword=def", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "proxyUsername without proxyPassword", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyUsername=abc", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "proxyPassword without proxyUsername", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyPassword=def", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "multiple proxyHost parameters", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyHost=localhost2", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "multiple proxyPort parameters", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyPort=1234&proxyPort=12345", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "multiple proxyUsername parameters", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyUsername=abc&proxyUsername=def&proxyPassword=123", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "multiple proxyPassword parameters", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyUsername=abc&proxyPassword=123&proxyPassword=456", + "valid": false, + "warning": false, + "hosts": null, + "auth": null, + "options": null + }, + { + "description": "only host present", + "uri": "mongodb://localhost/?proxyHost=localhost", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + }, + { + "description": "host and default port present", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyPort=1080", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + }, + { + "description": "host and non-default port present", + "uri": "mongodb://localhost/?proxyHost=localhost&proxyPort=12345", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + }, + { + "description": "replicaset, host and non-default port present", + "uri": "mongodb://rs1,rs2,rs3/?proxyHost=localhost&proxyPort=12345", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + }, + { + "description": "all options present", + "uri": "mongodb://rs1,rs2,rs3/?proxyHost=localhost&proxyPort=12345&proxyUsername=asdf&proxyPassword=qwerty", + "valid": true, + "warning": false, + "hosts": null, + "auth": null, + "options": {} + } + ] +} diff --git a/tools/synchro.py b/tools/synchro.py index 4d2f865479..3c79c284e9 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -140,6 +140,13 @@ "async_joinall": "joinall", "async_simple_test_client": "simple_test_client", "_async_create_connection": "_create_connection", + "_async_create_proxy_connection": "_create_proxy_connection", + "_async_socks5_handshake": "_socks5_handshake", + "_async_socks5_read_exact": "_socks5_read_exact", + "_async_socks5_sendall": "_socks5_sendall", + "_async_socks5_recv": "_socks5_recv", + "_async_proxy_connect": "_proxy_connect", + "_async_proxy_getaddrinfo": "_proxy_getaddrinfo", "pymongo.asynchronous.srv_resolver._SrvResolver.get_hosts": "pymongo.synchronous.srv_resolver._SrvResolver.get_hosts", "dns.asyncresolver.resolve": "dns.resolver.resolve", "__aenter__": "__enter__",