Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions src/apify_client/_streamed_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,14 +97,11 @@ def _process_whole_log(self, log: bytes | None) -> None:

@staticmethod
def _guess_log_level_from_message(message: str) -> int:
"""Guess the log level from the message."""
# Using only levels explicitly mentioned in the logging module
"""Read the level token after the platform timestamp and optional bracketed logger name."""
match = re.match(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z[ \t]+(?:\[[^\]\n]*\][ \t]+)?(\S+)', message)
known_levels = ('CRITICAL', 'FATAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET')
level_names_to_levels = logging.getLevelNamesMapping()
for level in known_levels:
if level in message:
return level_names_to_levels[level]
# Unknown log level. Fall back to the default.
if match and match[1] in known_levels:
return logging.getLevelNamesMapping()[match[1]]
return logging.INFO


Expand All @@ -113,7 +110,7 @@ class StreamedLog(StreamedLogBase):
"""Streams Actor run log output to a Python logger in a background thread.

The log stream is consumed in a background thread and each log message is forwarded to the provided logger with
an appropriate log level inferred from the message content.
the level token after the timestamp and optional bracketed logger name, defaulting to INFO for unknown levels.

Can be used as a context manager, which automatically starts and stops the streaming thread. Alternatively,
call `start` and `stop` manually. Obtain an instance via `RunClient.get_streamed_log`.
Expand Down Expand Up @@ -256,12 +253,15 @@ class StreamedLogAsync(StreamedLogBase):
"""Streams Actor run log output to a Python logger in an asyncio task.

The log stream is consumed in a background asyncio task and each log message is forwarded to the provided logger
with an appropriate log level inferred from the message content.
with the level token after the timestamp and optional bracketed logger name, defaulting to INFO for unknown levels.

Can be used as an async context manager, which automatically starts and cancels the streaming task. Alternatively,
call `start` and `stop` manually. Obtain an instance via `RunClientAsync.get_streamed_log`.
"""

_stop_timeout_s: ClassVar[float] = 1
"""Grace period for in-flight log data before the streaming task is cancelled."""

def __init__(self, log_client: LogClientAsync, *, to_logger: logging.Logger, from_start: bool = True) -> None:
"""Initialize `StreamedLogAsync`.

Expand All @@ -288,15 +288,28 @@ def start(self) -> Task:
return self._streaming_task

async def stop(self) -> None:
"""Stop the streaming task.
"""Wait up to one second for the stream to end, then cancel the streaming task.

Caller cancellation cancels the streaming task and propagates without reading the whole log.
If no stream has delivered anything yet, read the whole log in one request instead.
"""
if not self._streaming_task:
raise RuntimeError('Streaming task is not active')

was_streaming = not self._streaming_task.done()
self._streaming_task.cancel()
try:
await asyncio.wait_for(asyncio.shield(self._streaming_task), timeout=self._stop_timeout_s)
except TimeoutError:
self._streaming_task.cancel()
except asyncio.CancelledError:
self._streaming_task.cancel()
try:
await self._streaming_task
except asyncio.CancelledError:
pass
finally:
self._streaming_task = None
raise
try:
await self._streaming_task
except asyncio.CancelledError:
Expand Down
112 changes: 110 additions & 2 deletions tests/unit/test_streamed_log.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
from __future__ import annotations

import asyncio
import logging
from unittest.mock import Mock
from typing import TYPE_CHECKING
from unittest.mock import AsyncMock, Mock

import pytest

from apify_client._streamed_log import StreamedLogBase
from apify_client._streamed_log import StreamedLogAsync, StreamedLogBase

if TYPE_CHECKING:
from collections.abc import AsyncIterator


@pytest.mark.parametrize(
Expand Down Expand Up @@ -38,3 +43,106 @@ def test_streamed_log_redirects_each_complete_line(chunks: list[bytes], expected

calls = [(call.kwargs['level'], call.kwargs['msg']) for call in to_logger.log.call_args_list]
assert calls == expected_calls


@pytest.mark.parametrize(
('content', 'level'),
[
pytest.param('INFO Saved ERROR_PAGE screenshot', logging.INFO, id='info-with-error-text'),
pytest.param('DEBUG WARNING in payload', logging.DEBUG, id='debug-with-warning-text'),
pytest.param('ERROR failed\nCRITICAL detail', logging.ERROR, id='multiline'),
pytest.param('[apify] DEBUG ERROR in payload', logging.DEBUG, id='logger-prefix'),
pytest.param('WARNING warning', logging.WARNING, id='warning'),
pytest.param('FATAL failed', logging.FATAL, id='fatal'),
pytest.param('ERROR_PAGE saved', logging.INFO, id='unknown-token'),
pytest.param('Saved ERROR screenshot', logging.INFO, id='no-level'),
pytest.param('', logging.INFO, id='empty'),
],
)
def test_streamed_log_level_uses_token(content: str, level: int) -> None:
"""Only the level token immediately after the timestamp determines severity."""
logger = Mock(spec=logging.Logger)
stream = StreamedLogBase(logger)
stream._process_whole_log(f'2026-11-15T13:00:00.000Z {content}'.encode())
assert logger.log.call_args.kwargs['level'] == level


async def test_async_stop_drains_in_flight_chunk() -> None:
"""Stopping lets the last in-flight chunk reach the logger before the stream ends."""
delivered = asyncio.Event()
release = asyncio.Event()

async def chunks() -> AsyncIterator[bytes]:
yield b'2026-11-15T13:00:00.000Z INFO first\n'
delivered.set()
await release.wait()
yield b'2026-11-15T13:00:01.000Z INFO last\n'

response = Mock()
response.aiter_bytes.side_effect = chunks
context = AsyncMock()
context.__aenter__.return_value = response
client = Mock()
client.stream.return_value = context
logger = Mock(spec=logging.Logger)
stream = StreamedLogAsync(client, to_logger=logger)
task = stream.start()
await delivered.wait()
asyncio.get_running_loop().call_soon(release.set)
await stream.stop()
assert not task.cancelled()
assert len(logger.log.call_args_list) == 2
assert logger.log.call_args.kwargs['msg'].endswith('INFO last')


async def test_async_stop_cancels_silent_stream(monkeypatch: pytest.MonkeyPatch) -> None:
"""A stream that stays silent is cancelled after the grace period."""
logger = Mock(spec=logging.Logger)
stream = StreamedLogAsync(Mock(), to_logger=logger)
monkeypatch.setattr(StreamedLogAsync, '_stop_timeout_s', 0.01, raising=False)
stream._received_data = True
task = asyncio.create_task(asyncio.Event().wait())
stream._streaming_task = task
await stream.stop()
assert task.cancelled()
assert stream._streaming_task is None


async def test_async_stop_cancels_stream_when_caller_is_cancelled() -> None:
"""Cancelling stop during the grace period must not leave it waiting for the stream."""
stream = StreamedLogAsync(Mock(), to_logger=Mock(spec=logging.Logger))
stream._log_client.get_as_bytes = AsyncMock()
task = asyncio.create_task(asyncio.Event().wait())
stream._streaming_task = task
stopper = asyncio.create_task(stream.stop())
await asyncio.sleep(0)
stopper.cancel()
try:
done, _ = await asyncio.wait({stopper}, timeout=0.1)
assert stopper in done
assert stopper.cancelled()
stream._log_client.get_as_bytes.assert_not_awaited()
assert task.cancelled()
assert stream._streaming_task is None
finally:
task.cancel()
await asyncio.gather(task, stopper, return_exceptions=True)


async def test_async_stop_propagates_caller_timeout() -> None:
"""A caller timeout cancels the stream and skips the whole-log fallback."""
client = Mock()
client.get_as_bytes = AsyncMock()
stream = StreamedLogAsync(client, to_logger=Mock(spec=logging.Logger))
task = asyncio.create_task(asyncio.Event().wait())
stream._streaming_task = task
try:
with pytest.raises(TimeoutError):
async with asyncio.timeout(0.01):
await stream.stop()
assert task.cancelled()
assert stream._streaming_task is None
client.get_as_bytes.assert_not_awaited()
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
Loading