From 8eb702f20d6aac562d4b3d844e4b7548ff16593d Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Mon, 16 Nov 2026 17:45:48 +0100 Subject: [PATCH] fix: ensure streaming queries stay open and can be closed Signed-off-by: Gabor Boros --- rethinkdb/net.py | 4 +- rethinkdb/net_asyncio.py | 34 ++++++++++++----- rethinkdb/net_gevent.py | 29 ++++++++++---- rethinkdb/net_tornado.py | 38 ++++++++++++++++-- rethinkdb/net_trio.py | 31 +++++++++------ rethinkdb/net_twisted.py | 22 ++++++++++- tests/integration/test_net_asyncio.py | 55 +++++++++++++++++++++++++++ tests/integration/test_net_gevent.py | 34 +++++++++++++++++ tests/integration/test_net_tornado.py | 37 ++++++++++++++++++ tests/integration/test_net_trio.py | 36 ++++++++++++++++++ tests/integration/test_net_twisted.py | 51 +++++++++++++++++++++++++ 11 files changed, 335 insertions(+), 36 deletions(-) diff --git a/rethinkdb/net.py b/rethinkdb/net.py index 405301b..07df7e1 100644 --- a/rethinkdb/net.py +++ b/rethinkdb/net.py @@ -363,7 +363,9 @@ def raise_error(self, message: str): """ if self.error is None: self.error = ReqlRuntimeError(message, self.query.term_type, []) - dummy_response = f'{"t":{PbResponse.ResponseType.SUCCESS_SEQUENCE},"r":[]}' + dummy_response = ( + f'{{"t":{PbResponse.ResponseType.SUCCESS_SEQUENCE},"r":[]}}' + ).encode() self.extend(dummy_response) def _maybe_fetch_batch(self): diff --git a/rethinkdb/net_asyncio.py b/rethinkdb/net_asyncio.py index fe3212b..59e9ed2 100644 --- a/rethinkdb/net_asyncio.py +++ b/rethinkdb/net_asyncio.py @@ -92,8 +92,8 @@ class AsyncioCursor(Cursor): """The asyncio implementation of the Cursor object.""" def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) self.new_response = asyncio.Future() + super().__init__(*args, **kwargs) def __aiter__(self): return self @@ -109,10 +109,14 @@ async def close(self): # pylint: disable=invalid-overridden-method self.error = self._empty_error() if self.conn.is_open(): self.outstanding_requests += 1 - await self.conn.parent._stop(self) # pylint: disable=protected-access + await self.conn.parent.stop(self) + + def extend(self, res_buf): + """Extend the cursor and wake tasks waiting for the next batch.""" + self._extend(res_buf) def _extend(self, res_buf): - super()._extend(res_buf) # pylint: disable=no-member + super().extend(res_buf) self.new_response.set_result(True) self.new_response = asyncio.Future() @@ -159,7 +163,7 @@ def _maybe_fetch_batch(self): self.outstanding_requests += 1 # Keep a reference so the task is not garbage-collected before it runs. self._fetch_batch_task = asyncio.ensure_future( - self.conn.parent._continue(self), # pylint: disable=protected-access + self.conn.parent.resume(self), loop=self.conn.io_loop, ) @@ -181,6 +185,21 @@ def __init__(self, parent, io_loop=None): if self._io_loop is None: self._io_loop = asyncio.get_event_loop() + @property + def parent(self): + """Return the parent connection.""" + return self._parent + + @property + def cursor_cache(self): + """Return the cursor cache.""" + return self._cursor_cache + + @property + def io_loop(self): + """Return the event loop used by this connection.""" + return self._io_loop + def client_port(self): """Get the client port of the connection.""" if self.is_open(): @@ -274,7 +293,7 @@ async def close(self, noreply_wait=False, token=None, exception=None): # Cursors may remove themselves when errored, so copy a list of them for cursor in list(self._cursor_cache.values()): - cursor._error(err_message) # pylint: disable=protected-access + cursor.raise_error(err_message) for _query, future in iter(self._user_queries.values()): if not future.done(): @@ -377,11 +396,6 @@ async def __aenter__(self): async def __aexit__(self, exception_type, exception_val, traceback): await self.close(False) - async def _stop(self, cursor): - self.check_open() - q = Query(PbQuery.QueryType.STOP, cursor.query.token, None, None) - return await self._instance.run_query(q, True) # pylint: disable=protected-access - async def reconnect( # pylint: disable=invalid-overridden-method self, noreply_wait=True, timeout=None ): diff --git a/rethinkdb/net_gevent.py b/rethinkdb/net_gevent.py index 1611737..d762041 100644 --- a/rethinkdb/net_gevent.py +++ b/rethinkdb/net_gevent.py @@ -44,25 +44,30 @@ class GeventCursor(Cursor): """Gevent-based cursor implementation for RethinkDB queries.""" def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) self.new_response = Event() + super().__init__(*args, **kwargs) def __iter__(self): return self def __next__(self): - return self._get_next(None) + try: + return self._get_next(None) + except ReqlCursorEmpty as exc: + raise StopIteration from exc @staticmethod def _empty_error(): """Return the empty error exception class.""" return ReqlCursorEmpty() + def extend(self, res_buf): + """Extend the cursor and wake greenlets waiting for the next batch.""" + self._extend(res_buf) + def _extend(self, res_buf): """Extend the cursor with new response data.""" - # pylint: disable=no-member - super()._extend(res_buf) - # pylint: enable=no-member + super().extend(res_buf) self.new_response.set() self.new_response.clear() @@ -226,6 +231,16 @@ def __init__(self, parent, io_loop=None): self._write_mutex = Semaphore() self._socket = None + @property + def parent(self): + """Return the parent connection.""" + return self._parent + + @property + def cursor_cache(self): + """Return the cursor cache.""" + return self._cursor_cache + def connect(self, timeout): """Connect to the RethinkDB server.""" with gevent.Timeout( @@ -251,9 +266,7 @@ def close(self, noreply_wait=False, token=None, exception=None): # Cursors may remove themselves when errored, so copy a list of them for cursor in list(self._cursor_cache.values()): - # pylint: disable=protected-access - cursor._error(err_message) - # pylint: enable=protected-access + cursor.raise_error(err_message) for _query, async_res in iter(self._user_queries.values()): async_res.set_exception(ReqlDriverError(err_message)) diff --git a/rethinkdb/net_tornado.py b/rethinkdb/net_tornado.py index 34fa583..1417160 100644 --- a/rethinkdb/net_tornado.py +++ b/rethinkdb/net_tornado.py @@ -73,10 +73,17 @@ def __aiter__(self): async def __anext__(self): """Get the next item from the cursor.""" - if await self.fetch_next(): - return await self.next() - - raise StopAsyncIteration + try: + if not await self.fetch_next(): + raise StopAsyncIteration + # next() returns the buffered row directly. It is only awaitable when + # a subclass waits for the next batch inside _get_next. + item = self.next() + if asyncio.iscoroutine(item) or asyncio.isfuture(item): + return await item + return item + except ReqlCursorEmpty as exc: + raise StopAsyncIteration from exc def extend(self, res_buf): Cursor.extend(self, res_buf) @@ -120,6 +127,19 @@ def _get_next(self, timeout=None): ) return self.items.popleft() + def _maybe_fetch_batch(self): + if ( + self.error is None + and len(self.items) < self.threshold + and self.outstanding_requests == 0 + ): + self.outstanding_requests += 1 + # resume() is a coroutine; the IOLoop must run it so CONTINUE is sent. + self.conn._io_loop.spawn_callback( # pylint: disable=protected-access + self.conn.parent.resume, + self, + ) + class ConnectionInstance: """ @@ -137,6 +157,16 @@ def __init__(self, parent: "Connection", io_loop: IOLoop | None = None): self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) self._socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + @property + def parent(self) -> "Connection": + """Return the parent connection.""" + return self._parent + + @property + def cursor_cache(self) -> dict[int, TornadoCursor]: + """Return the cursor cache.""" + return self._cursor_cache + def client_port(self) -> int | None: """ Get the client port of the connection. diff --git a/rethinkdb/net_trio.py b/rethinkdb/net_trio.py index 18fae8e..796a202 100644 --- a/rethinkdb/net_trio.py +++ b/rethinkdb/net_trio.py @@ -137,13 +137,13 @@ async def close(self): # pylint: disable=invalid-overridden-method aclose = close + def extend(self, res_buf): + """Extend the cursor and wake tasks waiting for the next batch.""" + self._extend(res_buf) + def _extend(self, res_buf): - """Override so that we can make this async, and also to wake up blocked - tasks.""" - self.outstanding_requests -= 1 - self._maybe_fetch_batch() - res = Response(self.query.token, res_buf, self._json_decoder) - self._extend_internal(res) + """Append a response batch and wake tasks waiting for it.""" + super().extend(res_buf) self._new_response.set() self._new_response = trio.Event() @@ -179,8 +179,8 @@ async def _get_next(self, timeout=None): # pylint: disable=invalid-overridden-m return item async def _parent_continue(self): - # pylint: disable=protected-access - return await self.conn._parent._continue(self) + """Send a CONTINUE request for this cursor.""" + return await self.conn.parent.resume(self) def _maybe_fetch_batch(self): if ( @@ -189,8 +189,7 @@ def _maybe_fetch_batch(self): and self.outstanding_requests == 0 ): self.outstanding_requests += 1 - # pylint: disable=protected-access - self._nursery.start_soon(self.conn._parent._continue, self) + self._nursery.start_soon(self._parent_continue) class ConnectionInstance: @@ -206,6 +205,16 @@ def __init__(self, parent, nursery=None): self._reader_ended_event = None self._nursery = nursery + @property + def parent(self): + """Return the parent connection.""" + return self._parent + + @property + def cursor_cache(self): + """Return the cursor cache.""" + return self._cursor_cache + def client_port(self): if self.is_open(): return self._sockname[1] @@ -319,7 +328,7 @@ async def close(self, noreply_wait=False, token=None, exception=None): # Cursors may remove themselves when errored, so copy a list of them for cursor in list(self._cursor_cache.values()): - cursor._error(err_message) # pylint: disable=protected-access + cursor.raise_error(err_message) for _, future in self._user_queries.values(): if not future.done(): diff --git a/rethinkdb/net_twisted.py b/rethinkdb/net_twisted.py index ac98def..93b8ad2 100644 --- a/rethinkdb/net_twisted.py +++ b/rethinkdb/net_twisted.py @@ -30,6 +30,7 @@ from twisted.internet.endpoints import clientFromString from twisted.internet.error import TimeoutError as TwistedTimeoutError from twisted.internet.protocol import ClientFactory, Protocol +from twisted.python import log from rethinkdb.errors import ( ReqlAuthError, @@ -269,7 +270,8 @@ def _is_empty(self) -> bool: @inlineCallbacks def _get_next(self, timeout: float | None = None) -> "Deferred[Any]": if len(self.items) == 0 and self.error is not None: - return defer.fail(self.error) + result = yield defer.fail(self.error) + return result def raise_timeout(errback: Any) -> None: if isinstance(errback.value, CancelledError): @@ -284,7 +286,18 @@ def raise_timeout(errback: Any) -> None: reactor.callLater(timeout, item_defer.cancel) self._maybe_fetch_batch() - return item_defer + item = yield item_defer + return item + + def _maybe_fetch_batch(self) -> None: + if ( + self.error is None + and len(self.items) < self.threshold + and self.outstanding_requests == 0 + ): + self.outstanding_requests += 1 + deferred = self.conn.parent.resume(self) + deferred.addErrback(log.err) class ConnectionInstance: @@ -397,6 +410,11 @@ def connect(self, timeout: int) -> Generator: return self._parent + @property + def parent(self) -> "Connection": + """Return the parent connection.""" + return self._parent + @property def cursor_cache(self) -> dict[int, "TwistedCursor"]: """ diff --git a/tests/integration/test_net_asyncio.py b/tests/integration/test_net_asyncio.py index 6f544f3..29eda1a 100644 --- a/tests/integration/test_net_asyncio.py +++ b/tests/integration/test_net_asyncio.py @@ -1,3 +1,5 @@ +import asyncio +import gc import os import pytest @@ -24,6 +26,7 @@ def cleanup_test_dbs(): yield test_dbs = [ + "test_asyncio_between_db", "test_asyncio_cursor_db", "test_asyncio_helpers_db", "test_asyncio_db", @@ -174,6 +177,58 @@ async def test_asyncio_cursor_operations(conn): pass +@pytest.mark.asyncio +@pytest.mark.integration +async def test_asyncio_between_secondary_index(conn): + """A secondary-index between query returns rows and close finishes the reader.""" + connection = await conn + assert connection.is_open(), "Connection should be open" + + db_name = "test_asyncio_between_db" + table_name = "station" + loop = asyncio.get_running_loop() + previous_handler = loop.get_exception_handler() + messages: list[str] = [] + + def handler(running_loop, context): + messages.append(str(context.get("message", ""))) + if previous_handler is not None: + previous_handler(running_loop, context) + else: + running_loop.default_exception_handler(context) + + loop.set_exception_handler(handler) + try: + try: + await connection.start(query.db_drop(db_name)) + except Exception: + pass + + await connection.start(query.db_create(db_name)) + await connection.start(query.db(db_name).table_create(table_name)) + table = query.db(db_name).table(table_name) + docs = [{"id": i, "n": i} for i in range(5)] + inserted = await connection.start(table.insert(docs)) + assert inserted["inserted"] == 5 + await connection.start(table.index_create("n")) + await connection.start(table.index_wait("n")) + + cursor = await connection.start( + table.between(0, 4, index="n", right_bound="closed") + ) + results = [item async for item in cursor] + assert sorted(row["id"] for row in results) == [0, 1, 2, 3, 4] + + await connection.start(query.db_drop(db_name)) + await connection.close() + gc.collect() + assert not any( + "Task was destroyed but it is pending" in message for message in messages + ) + finally: + loop.set_exception_handler(previous_handler) + + @pytest.mark.asyncio @pytest.mark.integration async def test_asyncio_error_handling(conn): diff --git a/tests/integration/test_net_gevent.py b/tests/integration/test_net_gevent.py index 70da5e5..e6fe519 100644 --- a/tests/integration/test_net_gevent.py +++ b/tests/integration/test_net_gevent.py @@ -24,6 +24,7 @@ def cleanup_test_dbs(): yield test_dbs = [ + "test_gevent_between_db", "test_gevent_cursor_db", "test_gevent_helpers_db", "test_gevent_db", @@ -170,6 +171,39 @@ def test_gevent_cursor_operations(conn): pass +@pytest.mark.gevent +@pytest.mark.integration +def test_gevent_between_secondary_index(conn): + """A secondary-index between query returns every row and the connection closes.""" + connection = conn + assert connection.is_open(), "Connection should be open" + + db_name = "test_gevent_between_db" + table_name = "station" + + try: + connection.start(query.db_drop(db_name)) + except Exception: + pass + + connection.start(query.db_create(db_name)) + connection.start(query.db(db_name).table_create(table_name)) + table = query.db(db_name).table(table_name) + docs = [{"id": i, "n": i} for i in range(5)] + inserted = connection.start(table.insert(docs)) + assert inserted["inserted"] == 5 + connection.start(table.index_create("n")) + connection.start(table.index_wait("n")) + + cursor = connection.start(table.between(0, 4, index="n", right_bound="closed")) + results = list(cursor) + assert sorted(row["id"] for row in results) == [0, 1, 2, 3, 4] + + connection.start(query.db_drop(db_name)) + connection.close() + assert connection.is_open() is False + + @pytest.mark.gevent @pytest.mark.integration def test_gevent_error_handling(conn): diff --git a/tests/integration/test_net_tornado.py b/tests/integration/test_net_tornado.py index 64f5220..e0ba153 100644 --- a/tests/integration/test_net_tornado.py +++ b/tests/integration/test_net_tornado.py @@ -24,6 +24,7 @@ def cleanup_test_dbs(): yield test_dbs = [ + "test_tornado_between_db", "test_tornado_cursor_db", "test_tornado_helpers_db", "test_tornado_db", @@ -178,6 +179,42 @@ async def test_tornado_cursor_operations(conn): pass +@pytest.mark.tornado +@pytest.mark.integration +@pytest.mark.asyncio +async def test_tornado_between_secondary_index(conn): + """A secondary-index between query returns every row and the connection closes.""" + connection = await conn + assert connection.is_open(), "Connection should be open" + + db_name = "test_tornado_between_db" + table_name = "station" + + try: + await connection.start(query.db_drop(db_name)) + except Exception: + pass + + await connection.start(query.db_create(db_name)) + await connection.start(query.db(db_name).table_create(table_name)) + table = query.db(db_name).table(table_name) + docs = [{"id": i, "n": i} for i in range(5)] + inserted = await connection.start(table.insert(docs)) + assert inserted["inserted"] == 5 + await connection.start(table.index_create("n")) + await connection.start(table.index_wait("n")) + + cursor = await connection.start( + table.between(0, 4, index="n", right_bound="closed") + ) + results = [item async for item in cursor] + assert sorted(row["id"] for row in results) == [0, 1, 2, 3, 4] + + await connection.start(query.db_drop(db_name)) + await connection.close() + assert connection.is_open() is False + + @pytest.mark.tornado @pytest.mark.integration @pytest.mark.asyncio diff --git a/tests/integration/test_net_trio.py b/tests/integration/test_net_trio.py index 81a937f..b492e0e 100644 --- a/tests/integration/test_net_trio.py +++ b/tests/integration/test_net_trio.py @@ -25,6 +25,7 @@ def cleanup_test_dbs(): yield test_dbs = [ + "test_trio_between_db", "test_trio_cursor_db", "test_trio_helpers_db", "test_trio_db", @@ -176,6 +177,41 @@ async def test_trio_cursor_operations(conn): pass +@pytest.mark.trio +@pytest.mark.integration +async def test_trio_between_secondary_index(conn): + """A secondary-index between query returns every row and the connection closes.""" + connection = await conn + assert connection.is_open(), "Connection should be open" + + db_name = "test_trio_between_db" + table_name = "station" + + try: + await connection.start(query.db_drop(db_name)) + except Exception: + pass + + await connection.start(query.db_create(db_name)) + await connection.start(query.db(db_name).table_create(table_name)) + table = query.db(db_name).table(table_name) + docs = [{"id": i, "n": i} for i in range(5)] + inserted = await connection.start(table.insert(docs)) + assert inserted["inserted"] == 5 + await connection.start(table.index_create("n")) + await connection.start(table.index_wait("n")) + + cursor = await connection.start( + table.between(0, 4, index="n", right_bound="closed") + ) + results = [item async for item in cursor] + assert sorted(row["id"] for row in results) == [0, 1, 2, 3, 4] + + await connection.start(query.db_drop(db_name)) + await connection.close() + assert connection.is_open() is False + + @pytest.mark.trio @pytest.mark.integration async def test_trio_error_handling(conn): diff --git a/tests/integration/test_net_twisted.py b/tests/integration/test_net_twisted.py index 31c579b..517bec3 100644 --- a/tests/integration/test_net_twisted.py +++ b/tests/integration/test_net_twisted.py @@ -24,6 +24,7 @@ def cleanup_test_dbs(): yield test_dbs = [ + "test_twisted_between_db", "test_twisted_cursor_db", "test_twisted_helpers_db", "test_twisted_db", @@ -194,6 +195,56 @@ def run_test(): return run_test() +@pytest.mark.twisted +@pytest.mark.integration +def test_twisted_between_secondary_index(conn): + """A secondary-index between query returns every row and the connection closes.""" + + @defer.inlineCallbacks + def run_test(): + connection = yield conn + assert connection.is_open(), "Connection should be open" + + db_name = "test_twisted_between_db" + table_name = "station" + + try: + yield connection.start(query.db_drop(db_name)) + except Exception: + pass + + yield connection.start(query.db_create(db_name)) + yield connection.start(query.db(db_name).table_create(table_name)) + table = query.db(db_name).table(table_name) + docs = [{"id": i, "n": i} for i in range(5)] + inserted = yield connection.start(table.insert(docs)) + assert inserted["inserted"] == 5 + yield connection.start(table.index_create("n")) + yield connection.start(table.index_wait("n")) + + cursor = yield connection.start( + table.between(0, 4, index="n", right_bound="closed") + ) + results = [] + error = None + try: + while True: + has_next = yield cursor.fetch_next() + if not has_next: + break + item = yield cursor.next() + results.append(item) + except Exception as exc: + error = exc + + assert sorted(row["id"] for row in results) == [0, 1, 2, 3, 4], error + yield connection.start(query.db_drop(db_name)) + yield connection.close() + assert connection.is_open() is False + + return run_test() + + @pytest.mark.twisted @pytest.mark.integration def test_twisted_error_handling(conn):