Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions doc/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,18 @@ Bug fixes
due to a CPython memory leak bug (`python/cpython#116810 <https://github.com/python/cpython/issues/116810>`_).
The bug was fixed in Python 3.12.7 and Python 3.13.1.
Users of Python 3.12 and 3.13 should upgrade to at least those versions (`PYTHON-6182`_).
- Fixed a bug where ``AsyncMongoClient`` could not recover after a replica set
member removed itself from the topology with its own ``hello`` response (for
example, during a node restart): the monitor's executor cancelled its own
task, so the monitor never ran again (`PYTHON-6226`_).

.. _PYTHON-5929: https://jira.mongodb.org/browse/PYTHON-5929
.. _PYTHON-6172: https://jira.mongodb.org/browse/PYTHON-6172
.. _PYTHON-6149: https://jira.mongodb.org/browse/PYTHON-6149
.. _PYTHON-5873: https://jira.mongodb.org/browse/PYTHON-5873
.. _PYTHON-6145: https://jira.mongodb.org/browse/PYTHON-6145
.. _PYTHON-6182: https://jira.mongodb.org/browse/PYTHON-6182
.. _PYTHON-6226: https://jira.mongodb.org/browse/PYTHON-6226

Changes in Version 4.19.0 (2026/10/14)
--------------------------------------
Expand Down
8 changes: 6 additions & 2 deletions pymongo/asynchronous/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,12 @@ async def close(self) -> None:
"""
self._sdam.server_closed(self._description.address)

await self._monitor.close()
await self._pool.close()
try:
await self._monitor.close()
finally:
# The pool must close even if cancellation or an error
# interrupts the monitor close (PYTHON-6226).
await self._pool.close()

def request_check(self) -> None:
"""Check the server's state soon."""
Expand Down
5 changes: 4 additions & 1 deletion pymongo/asynchronous/topology.py
Original file line number Diff line number Diff line change
Expand Up @@ -704,10 +704,13 @@ async def _update_servers(self) -> None:

for address, server in list(self._servers.items()):
if not self._description.has_server(address):
await server.close()
# Update our state before closing: close() can cancel the
# calling task, so the stale server must be recorded as
# removed before close() runs, not after (PYTHON-6226).
if not _IS_SYNC:
self._monitor_tasks.append(server._monitor)
self._servers.pop(address)
await server.close()
Comment thread
blink1073 marked this conversation as resolved.
Resolved

async def cleanup_monitors(self) -> None:
tasks = []
Expand Down
11 changes: 10 additions & 1 deletion pymongo/periodic_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,16 @@ def close(self, dummy: Any = None) -> None:
callback; see monitor.py.
"""
self._stopped = True
if self._task is not None:
if self._task is None:
return
try:
current = asyncio.current_task()
except RuntimeError:
# close() also runs as a weakref callback, off the event loop.
current = None
# Never cancel the calling task: self-cancellation aborts close()
# mid-way and leaves the monitor dead (PYTHON-6226).
if self._task is not current:
self._task.cancel()

async def join(self, timeout: Optional[int] = None) -> None:
Expand Down
8 changes: 6 additions & 2 deletions pymongo/synchronous/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,12 @@ def close(self) -> None:
"""
self._sdam.server_closed(self._description.address)

self._monitor.close()
self._pool.close()
try:
self._monitor.close()
finally:
# The pool must close even if cancellation or an error
# interrupts the monitor close (PYTHON-6226).
self._pool.close()

def request_check(self) -> None:
"""Check the server's state soon."""
Expand Down
5 changes: 4 additions & 1 deletion pymongo/synchronous/topology.py
Original file line number Diff line number Diff line change
Expand Up @@ -702,10 +702,13 @@ def _update_servers(self) -> None:

for address, server in list(self._servers.items()):
if not self._description.has_server(address):
server.close()
# Update our state before closing: close() can cancel the
# calling task, so the stale server must be recorded as
# removed before close() runs, not after (PYTHON-6226).
if not _IS_SYNC:
self._monitor_tasks.append(server._monitor)
self._servers.pop(address)
server.close()

def cleanup_monitors(self) -> None:
tasks = []
Expand Down
157 changes: 156 additions & 1 deletion test/asynchronous/test_discovery_and_monitoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@
import os
import platform
import socketserver
import struct
import sys
import threading
import time
import unittest
from asyncio import StreamReader, StreamWriter
from pathlib import Path
from typing import Any

from pymongo.asynchronous.pool import AsyncConnection
from pymongo.errors import ConnectionFailure
Expand All @@ -38,6 +41,8 @@
from unittest.mock import patch

from bson import Timestamp, json_util
from bson import decode as bson_decode
from bson import encode as bson_encode
from pymongo import common, monitoring
from pymongo.asynchronous.settings import TopologySettings
from pymongo.asynchronous.topology import Topology
Expand All @@ -58,7 +63,7 @@
ServerHeartbeatStartedEvent,
)
from pymongo.server_description import SERVER_TYPE, ServerDescription
from pymongo.topology_description import TOPOLOGY_TYPE
from pymongo.topology_description import TOPOLOGY_TYPE, TopologyDescription
from pymongo.topology_shared import _ErrorContext
from test.asynchronous import (
AsyncIntegrationTest,
Expand Down Expand Up @@ -679,6 +684,156 @@ async def handle_client(reader: StreamReader, writer: StreamWriter):
)


class TestMonitorRecoversAfterSelfRemoval(AsyncUnitTest):
@unittest.skipUnless(not _IS_SYNC, "PYTHON-6226: monitor task self-cancellation is async-only")
async def test_monitor_recovers_after_self_removal(self):
# Regression test for PYTHON-6226: a monitor that closed its own
# server used to cancel itself, leaving the client stuck in
# ReplicaSetNoPrimary. The client must recover.

# Fake two-node replica set on ephemeral ports.
role: dict[int, str] = {}
hosts: list[str] = []

def make_handler(port_box):
async def handle(reader: StreamReader, writer: StreamWriter):
port = port_box["port"]
try:
while True:
length, req_id, _, _ = struct.unpack(
# The ignore is for the sync mirror, where
# unasync turns this read into a bare call.
"<iiii",
await reader.readexactly(16), # type: ignore[arg-type]
)
body = await reader.readexactly(length - 16)
cmd = next(iter(bson_decode(body[5:]))).lower()
if cmd in ("hello", "ismaster"):
reply: dict[str, Any] = {
"ok": 1,
"minWireVersion": 0,
"maxWireVersion": 21,
}
if role[port] != "standalone":
reply |= {
"setName": "rs",
"hosts": hosts,
"me": f"127.0.0.1:{port}",
"primary": hosts[1],
"secondary": role[port] == "secondary",
"isWritablePrimary": role[port] == "primary",
}
else:
reply = {"ok": 1}
data = b"\0" * 5 + bson_encode(reply)
writer.write(struct.pack("<iiii", 16 + len(data), 0, req_id, 2013) + data)
except (asyncio.IncompleteReadError, ConnectionError):
writer.close()

return handle

servers = []
ports = []
for _ in range(2):
port_box: dict[str, int] = {}
server = await asyncio.start_server(make_handler(port_box), "127.0.0.1", 0)
port_box["port"] = server.sockets[0].getsockname()[1]
ports.append(port_box["port"])
servers.append(server)

hosts[:] = [f"127.0.0.1:{p}" for p in ports]
role[ports[0]] = "secondary"
role[ports[1]] = "primary"

client = self.simple_client(
f"mongodb://{','.join(hosts)}",
replicaSet="rs",
heartbeatFrequencyMS=500, # driver minimum
serverMonitoringMode="poll",
serverSelectionTimeoutMS=2000,
)
try:
await client.admin.command("ping")

# Each node briefly removes itself from the topology, then
# recovers.
for port, original in list(role.items()):
role[port] = "standalone"
await asyncio.sleep(2.0)
role[port] = original
await asyncio.sleep(2.0)

# Both monitors self-removed; the client must recover.
await client.admin.command("ping")
finally:
# Close the client first: on Python 3.12+ Server.wait_closed()
# waits for accepted connections to disconnect (PYTHON-6226).
await client.close()
for server in servers:
server.close()
await asyncio.gather(*[server.wait_closed() for server in servers])


class TestServerRemovalHardening(AsyncUnitTest):
@unittest.skipUnless(not _IS_SYNC, "PYTHON-6226: task cancellation is async-only")
async def test_removal_bookkeeping_survives_close_cancellation(self):
# Hardening for PYTHON-6226: record the monitor and pop the server
# before close(), so cancellation during close() cannot strand it.

topology = await create_mock_topology("mongodb://a:27017,b:27017")
address_b = ("b", 27017)
victim = topology._servers[address_b]

async def cancelling_close():
# Simulate an executor cancelling the task that closes it.
asyncio.current_task().cancel()
await asyncio.sleep(0)

# A description that no longer contains the victim.
descriptions = topology._description.server_descriptions()
topology._description = TopologyDescription(
TOPOLOGY_TYPE.Unknown,
{addr: sd for addr, sd in descriptions.items() if addr != address_b},
None,
None,
None,
topology._settings,
)

async def remove_victim():
async with topology._lock:
with patch.object(victim, "close", cancelling_close):
await topology._update_servers()

task = asyncio.create_task(remove_victim())
with self.assertRaises(asyncio.CancelledError):
await task

self.assertNotIn(address_b, topology._servers)
self.assertIn(victim._monitor, topology._monitor_tasks)

@unittest.skipUnless(not _IS_SYNC, "PYTHON-6226: task cancellation is async-only")
async def test_pool_closes_when_monitor_close_is_cancelled(self):
# A cancelled monitor close must not leave the application pool open.

topology = await create_mock_topology("mongodb://a:27017,b:27017")
victim = topology._servers[("a", 27017)]

async def cancelling_monitor_close():
asyncio.current_task().cancel()
await asyncio.sleep(0)

async def close_victim():
with patch.object(victim._monitor, "close", cancelling_monitor_close):
await victim.close()

task = asyncio.create_task(close_victim())
with self.assertRaises(asyncio.CancelledError):
await task

self.assertTrue(victim.pool.closed)


# Generate unified tests.
globals().update(generate_test_classes(os.path.join(SDAM_PATH, "unified"), module=__name__))

Expand Down
88 changes: 87 additions & 1 deletion test/asynchronous/test_periodic_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import textwrap
import threading
import time
from typing import Any
from typing import Any, Optional
from unittest import mock

_interpreters: Any = None
Expand Down Expand Up @@ -155,6 +155,92 @@ async def target():
await executor.join(timeout=2)
self.assertGreaterEqual(called, 2)

async def test_close_from_own_task_does_not_cancel(self):
# Closing the executor from its own task must not cancel it
# (PYTHON-6226).
runs = 0

async def target():
nonlocal runs
executor.close()
# Raises CancelledError here if close() cancelled this task.
await asyncio.sleep(0)
runs += 1
return False

executor = self._make_executor(target=target)
executor.open()
await executor.join(timeout=2)
self.assertEqual(runs, 1, "target did not finish after close() from its own task")
if not _IS_SYNC:
assert executor._task is not None
self.assertFalse(executor._task.cancelled(), "executor task cancelled itself")
# The executor can be restarted after a self-close.
executor.open()
await executor.join(timeout=2)
self.assertEqual(runs, 2, "executor did not restart after a self-close")

async def test_close_from_thread_without_event_loop(self):
# close() can run off the event loop as a weakref callback, where
# asyncio.current_task() raises RuntimeError.
if _IS_SYNC:
started = threading.Event()
else:
started = asyncio.Event()

async def target():
started.set()
return True

executor = self._make_executor(min_interval=0.5, target=target)
executor.open()
if _IS_SYNC:
started.wait(timeout=2)
else:
assert isinstance(started, asyncio.Event)
await asyncio.wait_for(started.wait(), timeout=2)

close_error: Optional[BaseException] = None

def close_off_loop():
nonlocal close_error
try:
executor.close()
except BaseException as exc:
close_error = exc

real_task: Any = None
mock_task: Any = None
if not _IS_SYNC:
# Cancel a mock off the loop: cancelling the real task from
# another thread is not thread-safe under asyncio debug mode.
assert executor._task is not None
real_task = executor._task
mock_task = mock.MagicMock()
mock_task.cancelling.return_value = 0
executor._task = mock_task
try:
thread = threading.Thread(target=close_off_loop)
thread.start()
if _IS_SYNC:
thread.join(timeout=2)
else:
# Do not block the event loop while joining the thread.
await asyncio.to_thread(thread.join, 2)
finally:
if not _IS_SYNC:
executor._task = real_task
self.assertIsNone(close_error, "close() raised from a thread without a running event loop")
if not _IS_SYNC:
assert mock_task is not None and real_task is not None
mock_task.cancel.assert_called_once()
# Cancel the real executor on the event loop thread.
real_task.cancel()
await executor.join(timeout=2)
self.assertTrue(executor._stopped)
if not _IS_SYNC:
self.assertTrue(real_task.cancelled())

async def test_target_exception_stops_executor(self):
call_count = 0

Expand Down
Loading
Loading