From b2a3619fd3f8d90a23562a0e2ee933332ac87892 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sun, 15 Nov 2026 12:32:06 -0400 Subject: [PATCH 1/3] fix!: preserve streamed log severity and drain async logs on stop Read the level token after the platform timestamp and optional logger prefix. Allow in-flight asynchronous log chunks to finish for one second before cancellation. BREAKING CHANGE: Redirected severity follows the level token rather than message substrings; async stop may wait up to one second before cancelling. --- src/apify_client/_streamed_log.py | 27 +++++++----- tests/unit/test_streamed_log.py | 72 ++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/src/apify_client/_streamed_log.py b/src/apify_client/_streamed_log.py index f458c8e3..fe1e9877 100644 --- a/src/apify_client/_streamed_log.py +++ b/src/apify_client/_streamed_log.py @@ -97,14 +97,11 @@ def _process_whole_log(self, log: bytes | None) -> None: @staticmethod def _guess_log_level_from_message(message: str) -> int: - """Guess the log level from the message.""" - # Using only levels explicitly mentioned in the logging module + """Read the level token after the platform timestamp and optional bracketed logger name.""" + match = re.match(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z[ \t]+(?:\[[^\]\n]*\][ \t]+)?(\S+)', message) known_levels = ('CRITICAL', 'FATAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET') - level_names_to_levels = logging.getLevelNamesMapping() - for level in known_levels: - if level in message: - return level_names_to_levels[level] - # Unknown log level. Fall back to the default. + if match and match[1] in known_levels: + return logging.getLevelNamesMapping()[match[1]] return logging.INFO @@ -113,7 +110,7 @@ class StreamedLog(StreamedLogBase): """Streams Actor run log output to a Python logger in a background thread. The log stream is consumed in a background thread and each log message is forwarded to the provided logger with - an appropriate log level inferred from the message content. + the level token after the timestamp and optional bracketed logger name, defaulting to INFO for unknown levels. Can be used as a context manager, which automatically starts and stops the streaming thread. Alternatively, call `start` and `stop` manually. Obtain an instance via `RunClient.get_streamed_log`. @@ -256,12 +253,15 @@ class StreamedLogAsync(StreamedLogBase): """Streams Actor run log output to a Python logger in an asyncio task. The log stream is consumed in a background asyncio task and each log message is forwarded to the provided logger - with an appropriate log level inferred from the message content. + with the level token after the timestamp and optional bracketed logger name, defaulting to INFO for unknown levels. Can be used as an async context manager, which automatically starts and cancels the streaming task. Alternatively, call `start` and `stop` manually. Obtain an instance via `RunClientAsync.get_streamed_log`. """ + _stop_timeout_s: ClassVar[float] = 1 + """Grace period for in-flight log data before the streaming task is cancelled.""" + def __init__(self, log_client: LogClientAsync, *, to_logger: logging.Logger, from_start: bool = True) -> None: """Initialize `StreamedLogAsync`. @@ -288,7 +288,7 @@ def start(self) -> Task: return self._streaming_task async def stop(self) -> None: - """Stop the streaming task. + """Wait up to one second for the stream to end, then cancel the streaming task. If no stream has delivered anything yet, read the whole log in one request instead. """ @@ -296,7 +296,12 @@ async def stop(self) -> None: raise RuntimeError('Streaming task is not active') was_streaming = not self._streaming_task.done() - self._streaming_task.cancel() + try: + await asyncio.wait_for(asyncio.shield(self._streaming_task), timeout=self._stop_timeout_s) + except TimeoutError: + self._streaming_task.cancel() + except asyncio.CancelledError: + pass try: await self._streaming_task except asyncio.CancelledError: diff --git a/tests/unit/test_streamed_log.py b/tests/unit/test_streamed_log.py index f23ff9d0..625f683a 100644 --- a/tests/unit/test_streamed_log.py +++ b/tests/unit/test_streamed_log.py @@ -1,11 +1,16 @@ from __future__ import annotations +import asyncio import logging -from unittest.mock import Mock +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock, Mock import pytest -from apify_client._streamed_log import StreamedLogBase +from apify_client._streamed_log import StreamedLogAsync, StreamedLogBase + +if TYPE_CHECKING: + from collections.abc import AsyncIterator @pytest.mark.parametrize( @@ -38,3 +43,66 @@ def test_streamed_log_redirects_each_complete_line(chunks: list[bytes], expected calls = [(call.kwargs['level'], call.kwargs['msg']) for call in to_logger.log.call_args_list] assert calls == expected_calls + + +@pytest.mark.parametrize( + ('content', 'level'), + [ + pytest.param('INFO Saved ERROR_PAGE screenshot', logging.INFO, id='info-with-error-text'), + pytest.param('DEBUG WARNING in payload', logging.DEBUG, id='debug-with-warning-text'), + pytest.param('ERROR failed\nCRITICAL detail', logging.ERROR, id='multiline'), + pytest.param('[apify] DEBUG ERROR in payload', logging.DEBUG, id='logger-prefix'), + pytest.param('WARNING warning', logging.WARNING, id='warning'), + pytest.param('FATAL failed', logging.FATAL, id='fatal'), + pytest.param('ERROR_PAGE saved', logging.INFO, id='unknown-token'), + pytest.param('Saved ERROR screenshot', logging.INFO, id='no-level'), + pytest.param('', logging.INFO, id='empty'), + ], +) +def test_streamed_log_level_uses_token(content: str, level: int) -> None: + """Only the level token immediately after the timestamp determines severity.""" + logger = Mock(spec=logging.Logger) + stream = StreamedLogBase(logger) + stream._process_whole_log(f'2026-11-15T13:00:00.000Z {content}'.encode()) + assert logger.log.call_args.kwargs['level'] == level + + +async def test_async_stop_drains_in_flight_chunk() -> None: + """Stopping lets the last in-flight chunk reach the logger before the stream ends.""" + delivered = asyncio.Event() + release = asyncio.Event() + + async def chunks() -> AsyncIterator[bytes]: + yield b'2026-11-15T13:00:00.000Z INFO first\n' + delivered.set() + await release.wait() + yield b'2026-11-15T13:00:01.000Z INFO last\n' + + response = Mock() + response.aiter_bytes.side_effect = chunks + context = AsyncMock() + context.__aenter__.return_value = response + client = Mock() + client.stream.return_value = context + logger = Mock(spec=logging.Logger) + stream = StreamedLogAsync(client, to_logger=logger) + task = stream.start() + await delivered.wait() + asyncio.get_running_loop().call_soon(release.set) + await stream.stop() + assert not task.cancelled() + assert len(logger.log.call_args_list) == 2 + assert logger.log.call_args.kwargs['msg'].endswith('INFO last') + + +async def test_async_stop_cancels_silent_stream(monkeypatch: pytest.MonkeyPatch) -> None: + """A stream that stays silent is cancelled after the grace period.""" + logger = Mock(spec=logging.Logger) + stream = StreamedLogAsync(Mock(), to_logger=logger) + monkeypatch.setattr(StreamedLogAsync, '_stop_timeout_s', 0.01, raising=False) + stream._received_data = True + task = asyncio.create_task(asyncio.Event().wait()) + stream._streaming_task = task + await stream.stop() + assert task.cancelled() + assert stream._streaming_task is None From 5f4cdf707dec6177e50a2ae3f0803dcf5ee05cfd Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sun, 15 Nov 2026 12:48:27 -0400 Subject: [PATCH 2/3] fix: cancel streaming task when async stop is cancelled --- src/apify_client/_streamed_log.py | 4 +--- tests/unit/test_streamed_log.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/apify_client/_streamed_log.py b/src/apify_client/_streamed_log.py index fe1e9877..ffdd8de5 100644 --- a/src/apify_client/_streamed_log.py +++ b/src/apify_client/_streamed_log.py @@ -298,10 +298,8 @@ async def stop(self) -> None: was_streaming = not self._streaming_task.done() try: await asyncio.wait_for(asyncio.shield(self._streaming_task), timeout=self._stop_timeout_s) - except TimeoutError: + except (TimeoutError, asyncio.CancelledError): self._streaming_task.cancel() - except asyncio.CancelledError: - pass try: await self._streaming_task except asyncio.CancelledError: diff --git a/tests/unit/test_streamed_log.py b/tests/unit/test_streamed_log.py index 625f683a..30b38b0d 100644 --- a/tests/unit/test_streamed_log.py +++ b/tests/unit/test_streamed_log.py @@ -106,3 +106,22 @@ async def test_async_stop_cancels_silent_stream(monkeypatch: pytest.MonkeyPatch) await stream.stop() assert task.cancelled() assert stream._streaming_task is None + + +async def test_async_stop_cancels_stream_when_caller_is_cancelled() -> None: + """Cancelling stop during the grace period must not leave it waiting for the stream.""" + stream = StreamedLogAsync(Mock(), to_logger=Mock(spec=logging.Logger)) + stream._received_data = True + task = asyncio.create_task(asyncio.Event().wait()) + stream._streaming_task = task + stopper = asyncio.create_task(stream.stop()) + await asyncio.sleep(0) + stopper.cancel() + try: + done, _ = await asyncio.wait({stopper}, timeout=0.1) + assert stopper in done + assert task.cancelled() + assert stream._streaming_task is None + finally: + task.cancel() + await asyncio.gather(task, stopper, return_exceptions=True) From b58cfbf8c1d7dbea69e32f672769f01955c34b45 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sun, 15 Nov 2026 13:01:54 -0400 Subject: [PATCH 3/3] fix: propagate caller cancellation when stopping log streams --- src/apify_client/_streamed_log.py | 12 +++++++++++- tests/unit/test_streamed_log.py | 23 ++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/apify_client/_streamed_log.py b/src/apify_client/_streamed_log.py index ffdd8de5..6327e4c7 100644 --- a/src/apify_client/_streamed_log.py +++ b/src/apify_client/_streamed_log.py @@ -290,6 +290,7 @@ def start(self) -> Task: async def stop(self) -> None: """Wait up to one second for the stream to end, then cancel the streaming task. + Caller cancellation cancels the streaming task and propagates without reading the whole log. If no stream has delivered anything yet, read the whole log in one request instead. """ if not self._streaming_task: @@ -298,8 +299,17 @@ async def stop(self) -> None: was_streaming = not self._streaming_task.done() try: await asyncio.wait_for(asyncio.shield(self._streaming_task), timeout=self._stop_timeout_s) - except (TimeoutError, asyncio.CancelledError): + except TimeoutError: self._streaming_task.cancel() + except asyncio.CancelledError: + self._streaming_task.cancel() + try: + await self._streaming_task + except asyncio.CancelledError: + pass + finally: + self._streaming_task = None + raise try: await self._streaming_task except asyncio.CancelledError: diff --git a/tests/unit/test_streamed_log.py b/tests/unit/test_streamed_log.py index 30b38b0d..3cd197a1 100644 --- a/tests/unit/test_streamed_log.py +++ b/tests/unit/test_streamed_log.py @@ -111,7 +111,7 @@ async def test_async_stop_cancels_silent_stream(monkeypatch: pytest.MonkeyPatch) async def test_async_stop_cancels_stream_when_caller_is_cancelled() -> None: """Cancelling stop during the grace period must not leave it waiting for the stream.""" stream = StreamedLogAsync(Mock(), to_logger=Mock(spec=logging.Logger)) - stream._received_data = True + stream._log_client.get_as_bytes = AsyncMock() task = asyncio.create_task(asyncio.Event().wait()) stream._streaming_task = task stopper = asyncio.create_task(stream.stop()) @@ -120,8 +120,29 @@ async def test_async_stop_cancels_stream_when_caller_is_cancelled() -> None: try: done, _ = await asyncio.wait({stopper}, timeout=0.1) assert stopper in done + assert stopper.cancelled() + stream._log_client.get_as_bytes.assert_not_awaited() assert task.cancelled() assert stream._streaming_task is None finally: task.cancel() await asyncio.gather(task, stopper, return_exceptions=True) + + +async def test_async_stop_propagates_caller_timeout() -> None: + """A caller timeout cancels the stream and skips the whole-log fallback.""" + client = Mock() + client.get_as_bytes = AsyncMock() + stream = StreamedLogAsync(client, to_logger=Mock(spec=logging.Logger)) + task = asyncio.create_task(asyncio.Event().wait()) + stream._streaming_task = task + try: + with pytest.raises(TimeoutError): + async with asyncio.timeout(0.01): + await stream.stop() + assert task.cancelled() + assert stream._streaming_task is None + client.get_as_bytes.assert_not_awaited() + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True)