Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .evergreen/generated_configs/tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3778,6 +3778,36 @@ tasks:
- pr
- "9.0"

# Socks5 tests
- name: test-socks5-prose-noauth-nossl
commands:
- func: run server
vars:
TEST_NAME: socks5
VERSION: "9.0"
TOPOLOGY: replica_set
AUTH: noauth
SSL: nossl
- func: run tests
vars:
TEST_NAME: socks5
COVERAGE: "1"
tags: [socks5, pr]
- name: test-socks5-prose-noauth-ssl
commands:
- func: run server
vars:
TEST_NAME: socks5
VERSION: "9.0"
TOPOLOGY: replica_set
AUTH: noauth
SSL: ssl
- func: run tests
vars:
TEST_NAME: socks5
COVERAGE: "1"
tags: [socks5, pr]

# Standard tests
- name: test-standard-v4.4-python3.12-sync-noauth-ssl-replica-set
commands:
Expand Down
11 changes: 11 additions & 0 deletions .evergreen/generated_configs/variants.yml
Original file line number Diff line number Diff line change
Expand Up @@ -631,6 +631,17 @@ buildvariants:
TEST_NAME: sfp
tags: [pr]

# Socks5 tests
- name: socks5-prose
tasks:
- name: .socks5
display_name: SOCKS5 Prose Tests
run_on:
- rhel8.10-small
expansions:
UV_PYTHON: "3.11"
tags: [pr]

# Stable api tests
- name: stable-api-require-v1-rhel8-auth
tasks:
Expand Down
3 changes: 0 additions & 3 deletions .evergreen/remove-unimplemented-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,6 @@ find /$PYMONGO/test -type f -name 'pre-42-*.json' -delete
rm $PYMONGO/test/csot/override-collection-timeoutMS.json
rm $PYMONGO/test/csot/override-database-timeoutMS.json

# PYTHON-2943 - Socks5 Proxy Support
rm $PYMONGO/test/uri_options/proxy-options.json

# PYTHON-5517 - Avoid clearing the connection pool when the server connection rate limiter triggers
rm $PYMONGO/test/discovery_and_monitoring/unified/backpressure-*.json

Expand Down
35 changes: 35 additions & 0 deletions .evergreen/scripts/generate_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,19 @@ def create_coverage_report_variants():
return [create_variant(["coverage-report"], "Coverage Report", host=DEFAULT_HOST)]


def create_socks5_variants():
variant = create_variant(
[".socks5"],
"SOCKS5 Prose Tests",
host=DEFAULT_HOST,
tags=["pr"],
expansions=dict(UV_PYTHON=MIN_PYTHON),
patchable=True,
)
variant.name = "socks5-prose"
return [variant]


def create_kms_variants():
tasks = []
tasks.append(EvgTaskRef(name="test-gcpkms", batchtime=BATCHTIME_DAY))
Expand Down Expand Up @@ -972,6 +985,28 @@ def create_fuzz_tasks():
return tasks


def create_socks5_tasks():
tasks = []
for ssl in ["nossl", "ssl"]:
server_vars = dict(
TEST_NAME="socks5",
VERSION=MAX_VERSION,
TOPOLOGY="replica_set",
AUTH="noauth",
SSL=ssl,
)
commands = [
FunctionCall(func="run server", vars=server_vars),
FunctionCall(func="run tests", vars=dict(TEST_NAME="socks5", COVERAGE="1")),
]
tasks.append(
EvgTask(
name=f"test-socks5-prose-noauth-{ssl}", tags=["socks5", "pr"], commands=commands
)
)
return tasks


def create_kms_tasks():
tasks = []
for kms_type in ["gcp", "azure"]:
Expand Down
3 changes: 3 additions & 0 deletions .evergreen/scripts/run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,9 @@ def handle_fuzz() -> None:


def run() -> None:
if TEST_NAME == "socks5":
TEST_ARGS.extend(["test/test_socks5.py", "-k", "TestSocks5Prose"])

# Add diagnostic for python version.
print("Running with python", sys.version)

Expand Down
1 change: 0 additions & 1 deletion .evergreen/scripts/setup_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
"pyopenssl": "ocsp",
}


# Map the test name to test group.
GROUP_MAP = dict(fuzz="fuzz", mockupdb="mockupdb", perf="perf")

Expand Down
29 changes: 29 additions & 0 deletions .evergreen/scripts/test_generate_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,35 @@ def variant_selects(variant, task):
return False


def test_socks5_tasks_use_standard_setup_and_execution():
tasks = generate_config.create_socks5_tasks()
assert {task.name for task in tasks} == {
"test-socks5-prose-noauth-nossl",
"test-socks5-prose-noauth-ssl",
}
assert {task_vars(task)["SSL"] for task in tasks} == {"nossl", "ssl"}
for task in tasks:
assert {"socks5", "pr"} <= set(task.tags)
assert [command.func for command in task.commands] == ["run server", "run tests"]
assert task.commands[0].vars == dict(
TEST_NAME="socks5",
VERSION=generate_config_utils.MAX_VERSION,
TOPOLOGY="replica_set",
AUTH="noauth",
SSL=task_vars(task)["SSL"],
)
assert task.commands[1].vars == dict(TEST_NAME="socks5", COVERAGE="1")


def test_socks5_variant_selects_both_prose_tasks():
(variant,) = generate_config.create_socks5_variants()
assert variant.name == "socks5-prose"
assert variant.display_name == "SOCKS5 Prose Tests"
assert "pr" in variant.tags
assert variant.expansions["UV_PYTHON"] == generate_config_utils.MIN_PYTHON
assert all(variant_selects(variant, task) for task in generate_config.create_socks5_tasks())


def test_pr_tasks_pin_max_version():
"""Task-level pins: pr tasks pin MAX_VERSION (the "latest" nightly is
unstable); the explicit 4.4 min-deps pin is the exception. Variant-merged
Expand Down
1 change: 1 addition & 0 deletions .evergreen/scripts/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class Distro:
"ocsp": "ocsp",
"perf": "perf",
"sfp": "sfp",
"socks5": "",
"numpy": "",
}

Expand Down
10 changes: 10 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,16 @@ wheels (CPython 3.12-3.14).
then `just run-tests`. Set `FUZZ_MAX_TIME` (seconds, default 300) to change
the per-target budget.

## SOCKS5 proxy tests

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This whole entry isn't true, there's no change to setup-tests with socks5.


With a local replica set running and `MONGODB_URI` set to its connection string,
run `just setup-tests socks5`, then `just run-tests`, and finally
`just teardown-tests`. Setup requires the drivers-evergreen-tools checkout and
starts authenticated and unauthenticated proxies on ports 1080 and 1081. The
runner exercises the synchronous and asynchronous SOCKS5 prose tests; teardown
stops the proxies. For a TLS-enabled replica set, use
`just setup-tests socks5 --ssl` instead.

## Testing time-dependent operations

- `test.utils_shared.delay` - One can trigger an arbitrarily long-running operation on the server using this delay utility
Expand Down
8 changes: 8 additions & 0 deletions doc/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,13 @@ Changelog
Changes in Version 4.20.0 (2026/XX/XX)
--------------------------------------

- Added support for connecting to MongoDB and Client-Side Field Level
Encryption KMS servers through a SOCKS5 proxy via the ``proxyHost``,
``proxyPort``, ``proxyUsername``, and ``proxyPassword`` URI and keyword
options on :class:`~pymongo.synchronous.mongo_client.MongoClient` and
:class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient`
(`PYTHON-2943`_). SOCKS5 proxy options cannot be combined with
``kms_connect_callback``.
- The client handshake metadata now reports a detected coding agent in
``client.env.agent`` (`PYTHON-5929`_).

Expand Down Expand Up @@ -35,6 +42,7 @@ Bug fixes
The bug was fixed in Python 3.12.7 and Python 3.13.1.
Users of Python 3.12 and 3.13 should upgrade to at least those versions (`PYTHON-6182`_).

.. _PYTHON-2943: https://jira.mongodb.org/browse/PYTHON-2943
.. _PYTHON-5929: https://jira.mongodb.org/browse/PYTHON-5929
.. _PYTHON-6172: https://jira.mongodb.org/browse/PYTHON-6172
.. _PYTHON-6149: https://jira.mongodb.org/browse/PYTHON-6149
Expand Down
13 changes: 12 additions & 1 deletion pymongo/_kms_connect_shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,13 @@
import urllib.parse
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from typing import Any, Optional
from typing import TYPE_CHECKING, Any, Optional

from pymongo.errors import ConfigurationError

if TYPE_CHECKING:
from pymongo.pool_options import _ProxyOptions


@dataclass(frozen=True)
class KMSConnectContext:
Expand Down Expand Up @@ -71,6 +74,14 @@ class KMSConnectContext:
KMSConnectCallback = Callable[[KMSConnectContext], socket.socket]


def _check_kms_proxy_options(
proxy_options: Optional[_ProxyOptions],
kms_connect_callback: Optional[Callable[[KMSConnectContext], Any]],
) -> None:
if proxy_options is not None and kms_connect_callback is not None:
raise ConfigurationError("Cannot specify kms_connect_callback with proxyHost")


def _close_rejected_kms_socket(obj: Any) -> None:
"""Close a callback return value that failed validation, best effort.

Expand Down
3 changes: 2 additions & 1 deletion pymongo/asynchronous/_kms_connect.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ async def _connect_kms(
"""Connect to a KMS host and perform the TLS handshake over the socket.

Uses ``kms_connect_callback`` when one is provided, otherwise connects
directly, and always verifies against ``address`` (the KMS host).
using the configured SOCKS5 proxy or directly. Always verifies against
``address`` (the KMS host).
"""
if kms_connect_callback is None:
try:
Expand Down
31 changes: 26 additions & 5 deletions pymongo/asynchronous/encryption.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
from bson.codec_options import CodecOptions
from bson.raw_bson import DEFAULT_RAW_BSON_OPTIONS, RawBSONDocument, _inflate_bson
from pymongo import _csot, _op_id
from pymongo._kms_connect_shared import AsyncKMSConnectCallback
from pymongo._kms_connect_shared import AsyncKMSConnectCallback, _check_kms_proxy_options
from pymongo.asynchronous._kms_connect import _KMS_CONNECT_TIMEOUT, _connect_kms
from pymongo.asynchronous.collection import AsyncCollection
from pymongo.asynchronous.cursor import AsyncCursor
Expand Down Expand Up @@ -102,6 +102,8 @@
if TYPE_CHECKING:
from pymongocrypt.mongocrypt import MongoCryptKmsContext

from pymongo.pool_options import _ProxyOptions


_IS_SYNC = False

Expand All @@ -123,6 +125,7 @@ def __init__(
key_vault_coll: AsyncCollection[_DocumentTypeArg],
mongocryptd_client: Optional[AsyncMongoClient[_DocumentTypeArg]],
opts: AutoEncryptionOpts,
proxy_options: Optional[_ProxyOptions] = None,
):
"""Internal class to perform I/O on behalf of pymongocrypt."""
self.client_ref: Any
Expand All @@ -140,6 +143,7 @@ def __init__(
),
)
self.mongocryptd_client = mongocryptd_client
self._proxy_options = proxy_options
self.opts = opts
self._spawned = False
self._kms_ssl_contexts = opts._kms_ssl_contexts(_IS_SYNC)
Expand Down Expand Up @@ -182,6 +186,7 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None:
connect_timeout=connect_timeout,
socket_timeout=connect_timeout,
ssl_context=ctx,
proxy_options=self._proxy_options,
)
try:
conn = await _connect_kms(
Expand Down Expand Up @@ -360,6 +365,9 @@ def __init__(self, client: AsyncMongoClient[_DocumentTypeArg], opts: AutoEncrypt
:param client: The encrypted AsyncMongoClient.
:param opts: The encrypted client's :class:`AutoEncryptionOpts`.
"""
_check_kms_proxy_options(
client.options.pool_options._proxy_options, opts._kms_connect_callback
)
if opts._schema_map is None:
schema_map = None
else:
Expand Down Expand Up @@ -387,6 +395,7 @@ def _get_internal_client(
encrypter._internal_client = internal_client
return internal_client

# KMS requests use the encrypted client's SOCKS5 proxy options, not the key vault client's.
if opts._key_vault_client is not None:
key_vault_client = opts._key_vault_client
else:
Expand All @@ -409,6 +418,7 @@ def _get_internal_client(
key_vault_coll, # type:ignore[arg-type]
mongocryptd_client,
opts,
proxy_options=client.options.pool_options._proxy_options,
)
self._auto_encrypter = AsyncAutoEncrypter(
io_callbacks,
Expand Down Expand Up @@ -572,7 +582,8 @@ def __init__(
collection. Data keys are protected with encryption by a KMS
provider.
:param key_vault_client: An AsyncMongoClient connected to a MongoDB cluster
containing the `key_vault_namespace` collection.
containing the `key_vault_namespace` collection. Its SOCKS5 proxy
options are also used for KMS requests.
:param codec_options: An instance of
:class:`~bson.codec_options.CodecOptions` to use when encoding a
value for encryption and decoding the decrypted BSON value. This
Expand Down Expand Up @@ -604,8 +615,11 @@ def __init__(
deadline and closes any socket the callback yields later. For an
ordinary HTTP proxy, pass
:class:`~pymongo.encryption_options.AsyncHTTPProxyKMSConnect`.
Defaults to ``None``, meaning the driver connects to KMS hosts
directly.
Defaults to ``None``, meaning the driver uses the SOCKS5 proxy
configured on ``key_vault_client``, or connects directly if none
is configured. Cannot be combined with ``proxyHost`` on
``key_vault_client``; doing so raises
:class:`~pymongo.errors.ConfigurationError`.

.. versionchanged:: 4.19
Added the `kms_connect_callback` parameter.
Expand Down Expand Up @@ -639,6 +653,9 @@ def __init__(
f"AsyncMongoClient required but given {type(key_vault_client).__name__}"
)

_check_kms_proxy_options(
key_vault_client.options.pool_options._proxy_options, kms_connect_callback
)
self._kms_providers = kms_providers
self._key_vault_namespace = key_vault_namespace
self._key_vault_client = key_vault_client
Expand All @@ -656,7 +673,11 @@ def __init__(
)
self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC)
self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO(
None, key_vault_coll, None, opts
None,
key_vault_coll,
None,
opts,
proxy_options=key_vault_client.options.pool_options._proxy_options,
)
self._encryption = AsyncExplicitEncrypter(
self._io_callbacks,
Expand Down
Loading
Loading