From 6269d12ccb9ea126988649c135e5b5c87706fa61 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 11 Nov 2026 14:06:57 +0100 Subject: [PATCH 1/3] fix: extend the request timeout by `wait_for_finish` in `start()` and `build()` --- src/apify_client/_consts.py | 3 + .../_resource_clients/_resource_client.py | 17 +- src/apify_client/_resource_clients/actor.py | 30 +-- src/apify_client/_resource_clients/task.py | 10 +- src/apify_client/http_clients/_base.py | 21 +- tests/unit/test_wait_for_finish_timeout.py | 196 ++++++++++++++++++ 6 files changed, 251 insertions(+), 26 deletions(-) create mode 100644 tests/unit/test_wait_for_finish_timeout.py diff --git a/src/apify_client/_consts.py b/src/apify_client/_consts.py index ba73743a..5e6db6dd 100644 --- a/src/apify_client/_consts.py +++ b/src/apify_client/_consts.py @@ -32,6 +32,9 @@ DEFAULT_WAIT_WHEN_JOB_NOT_EXIST = timedelta(seconds=3) """How long to wait for a job to exist before giving up.""" +MAX_WAIT_FOR_FINISH_HOLD = timedelta(seconds=60) +"""Longest time the API holds a response for the `waitForFinish` parameter.""" + OVERRIDABLE_DEFAULT_HEADERS = {'Accept', 'Authorization', 'Accept-Encoding', 'User-Agent'} """Headers that can be overridden by users, but will trigger a warning if they do so, as it may lead to API errors.""" diff --git a/src/apify_client/_resource_clients/_resource_client.py b/src/apify_client/_resource_clients/_resource_client.py index f94ee829..77abeb57 100644 --- a/src/apify_client/_resource_clients/_resource_client.py +++ b/src/apify_client/_resource_clients/_resource_client.py @@ -7,7 +7,11 @@ from typing import TYPE_CHECKING, Any, Literal, get_args from urllib.parse import urlencode, urlparse, urlunparse -from apify_client._consts import DEFAULT_WAIT_FOR_FINISH, DEFAULT_WAIT_WHEN_JOB_NOT_EXIST +from apify_client._consts import ( + DEFAULT_WAIT_FOR_FINISH, + DEFAULT_WAIT_WHEN_JOB_NOT_EXIST, + MAX_WAIT_FOR_FINISH_HOLD, +) from apify_client._docs import docs_group from apify_client._logging import WithLogDetailsClient from apify_client._utils.errors import catch_not_found_for_resource_or_throw, catch_not_found_or_throw @@ -151,6 +155,17 @@ def _build_params(self, **kwargs: QueryParamValue) -> dict[str, QueryParamValue] merged = {**self._default_params, **kwargs} return {k: v for k, v in merged.items() if v is not None} + def _timeout_for_wait_for_finish(self, timeout: Timeout, wait_for_finish: int | None) -> Timeout: + """Extend a timeout tier by the time the API may hold the response for `wait_for_finish`. + + A request that times out while the API still holds it is retried, and a retried POST starts a second run or + build. An explicit `timedelta` and `'no_timeout'` are used as given. + """ + if isinstance(timeout, timedelta) or timeout == 'no_timeout' or wait_for_finish is None or wait_for_finish <= 0: + return timeout + hold = min(timedelta(seconds=wait_for_finish), MAX_WAIT_FOR_FINISH_HOLD) + return self._http_client._resolve_timeout(timeout) + hold # noqa: SLF001 + @staticmethod def _clean_json_payload(data: dict) -> dict: """Remove unset top-level fields from an API request payload. diff --git a/src/apify_client/_resource_clients/actor.py b/src/apify_client/_resource_clients/actor.py index 1f1d97d7..3a3d183d 100644 --- a/src/apify_client/_resource_clients/actor.py +++ b/src/apify_client/_resource_clients/actor.py @@ -333,7 +333,8 @@ def start( than the whole memory limit of the account is rejected with `actor-memory-limit-exceeded` as well and never starts, so `True` retries it forever. A streamed `run_input` that cannot be rewound, such as a generator, is sent only once, so its start is not retried. - timeout: Timeout for the API HTTP request. + timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the + request outlasts the time the API holds the response. Returns: The run object. @@ -360,7 +361,7 @@ def start( headers={'content-type': content_type}, data=run_input, params=request_params, - timeout=timeout, + timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish), ), wait_for_resources=wait_for_resources, ) @@ -497,7 +498,8 @@ def build( This is to enable quick rebuild during development. By default, the cache is not used. wait_for_finish: The maximum number of seconds the server waits for the build to finish before returning. By default it is 0, the maximum value is 60. - timeout: Timeout for the API HTTP request. + timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the + request outlasts the time the API holds the response. Returns: The build object. @@ -514,7 +516,7 @@ def build( url=self._build_url('builds'), method='POST', params=request_params, - timeout=timeout, + timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish), ) result = response_to_dict(response) @@ -547,7 +549,8 @@ def default_build( Args: wait_for_finish: The maximum number of seconds the server waits for the build to finish before returning. By default it is 0, the maximum value is 60. - timeout: Timeout for the API HTTP request. + timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the + request outlasts the time the API holds the response. Returns: The resource client for the default build of this Actor. @@ -560,7 +563,7 @@ def default_build( url=self._build_url('builds/default'), method='GET', params=request_params, - timeout=timeout, + timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish), ) result = response_to_dict(response) @@ -974,7 +977,8 @@ async def start( than the whole memory limit of the account is rejected with `actor-memory-limit-exceeded` as well and never starts, so `True` retries it forever. A streamed `run_input` that cannot be rewound, such as a generator, is sent only once, so its start is not retried. - timeout: Timeout for the API HTTP request. + timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the + request outlasts the time the API holds the response. Returns: The run object. @@ -1001,7 +1005,7 @@ async def start( headers={'content-type': content_type}, data=run_input, params=request_params, - timeout=timeout, + timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish), ), wait_for_resources=wait_for_resources, ) @@ -1139,7 +1143,8 @@ async def build( This is to enable quick rebuild during development. By default, the cache is not used. wait_for_finish: The maximum number of seconds the server waits for the build to finish before returning. By default it is 0, the maximum value is 60. - timeout: Timeout for the API HTTP request. + timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the + request outlasts the time the API holds the response. Returns: The build object. @@ -1156,7 +1161,7 @@ async def build( url=self._build_url('builds'), method='POST', params=request_params, - timeout=timeout, + timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish), ) result = response_to_dict(response) @@ -1189,7 +1194,8 @@ async def default_build( Args: wait_for_finish: The maximum number of seconds the server waits for the build to finish before returning. By default it is 0, the maximum value is 60. - timeout: Timeout for the API HTTP request. + timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the + request outlasts the time the API holds the response. Returns: The resource client for the default build of this Actor. @@ -1202,7 +1208,7 @@ async def default_build( url=self._build_url('builds/default'), method='GET', params=request_params, - timeout=timeout, + timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish), ) result = response_to_dict(response) diff --git a/src/apify_client/_resource_clients/task.py b/src/apify_client/_resource_clients/task.py index 20755701..162a0045 100644 --- a/src/apify_client/_resource_clients/task.py +++ b/src/apify_client/_resource_clients/task.py @@ -287,7 +287,8 @@ def start( `timedelta` stops retrying after that long and raises the last error. A run that requests more memory than the whole memory limit of the account is rejected with `actor-memory-limit-exceeded` as well and never starts, so `True` retries it forever. - timeout: Timeout for the API HTTP request. + timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the + request outlasts the time the API holds the response. Returns: The run object. @@ -313,7 +314,7 @@ def start( headers={'content-type': 'application/json; charset=utf-8'}, json=task_input.model_dump() if task_input is not None else None, params=request_params, - timeout=timeout, + timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish), ), wait_for_resources=wait_for_resources, ) @@ -724,7 +725,8 @@ async def start( `timedelta` stops retrying after that long and raises the last error. A run that requests more memory than the whole memory limit of the account is rejected with `actor-memory-limit-exceeded` as well and never starts, so `True` retries it forever. - timeout: Timeout for the API HTTP request. + timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the + request outlasts the time the API holds the response. Returns: The run object. @@ -750,7 +752,7 @@ async def start( headers={'content-type': 'application/json; charset=utf-8'}, json=task_input.model_dump() if task_input is not None else None, params=request_params, - timeout=timeout, + timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish), ), wait_for_resources=wait_for_resources, ) diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index efd79f07..568b8acf 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -11,7 +11,7 @@ from datetime import UTC, datetime, timedelta from http import HTTPStatus from importlib import metadata -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, Literal, TypeVar from urllib.parse import urlencode # `Protocol` comes from `typing_extensions`, not `typing`, because its runtime `isinstance` check looks attributes @@ -275,6 +275,16 @@ def _is_body_worth_compressing(data: str | bytes | bytearray | StreamedBodySourc return len(data) >= MIN_COMPRESSION_SIZE return False + def _resolve_timeout(self, timeout: timedelta | Literal['short', 'medium', 'long']) -> timedelta: + """Resolve a timeout tier to its configured duration. A `timedelta` is returned as given.""" + if timeout == 'short': + return self._timeout_short + if timeout == 'medium': + return self._timeout_medium + if timeout == 'long': + return self._timeout_long + return timeout + def _compute_timeout(self, timeout: Timeout, *, attempt: int) -> int | float | None: """Resolve a timeout tier and compute the timeout for a request attempt with exponential increase. @@ -292,14 +302,7 @@ def _compute_timeout(self, timeout: Timeout, *, attempt: int) -> int | float | N if timeout == 'no_timeout': return None - if timeout == 'short': - resolved = self._timeout_short - elif timeout == 'medium': - resolved = self._timeout_medium - elif timeout == 'long': - resolved = self._timeout_long - else: - resolved = timeout + resolved = self._resolve_timeout(timeout) if resolved > self._timeout_max: # Keyed per timeout kind, so retries and repeated calls do not spam the log with the same warning. diff --git a/tests/unit/test_wait_for_finish_timeout.py b/tests/unit/test_wait_for_finish_timeout.py new file mode 100644 index 00000000..aaa2ce28 --- /dev/null +++ b/tests/unit/test_wait_for_finish_timeout.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import inspect +import json +from dataclasses import dataclass +from datetime import timedelta +from typing import TYPE_CHECKING, Any +from unittest.mock import Mock + +import pytest +from werkzeug import Response + +from apify_client import ApifyClient, ApifyClientAsync + +if TYPE_CHECKING: + from collections.abc import Callable + + from pytest_httpserver import HTTPServer + + from apify_client.types import Timeout + +pytestmark = pytest.mark.usefixtures('http_client_classes') + +ACTOR_ID = 'test_actor_id' +TASK_ID = 'test_task_id' + +RUN_RESPONSE = { + 'data': { + 'id': 'test_run_id', + 'actId': ACTOR_ID, + 'userId': 'test_user_id', + 'startedAt': '2026-08-01T10:00:00.000Z', + 'status': 'RUNNING', + 'meta': {'origin': 'API'}, + 'stats': {'restartCount': 0, 'resurrectCount': 0, 'computeUnits': 0.1}, + 'options': {'build': 'latest', 'timeoutSecs': 300, 'memoryMbytes': 1024, 'diskMbytes': 2048}, + 'buildId': 'test_build_id', + 'generalAccess': 'RESTRICTED', + 'defaultKeyValueStoreId': 'test_kvs_id', + 'defaultDatasetId': 'test_dataset_id', + 'defaultRequestQueueId': 'test_rq_id', + 'buildNumber': '0.0.1', + 'containerUrl': 'https://test.runs.apify.net', + } +} + +BUILD_RESPONSE = { + 'data': { + 'id': 'test_build_id', + 'actId': ACTOR_ID, + 'userId': 'test_user_id', + 'startedAt': '2026-08-01T10:00:00.000Z', + 'status': 'RUNNING', + 'meta': {'origin': 'API'}, + 'buildNumber': '0.0.1', + } +} + + +@dataclass(frozen=True) +class CallSite: + """A client method that sends `waitForFinish`, with the endpoint it calls and its default timeout tier.""" + + invoke: Callable[..., Any] + method: str + path: str + response_body: dict + tier_secs: int + + +CALL_SITES = [ + pytest.param( + CallSite( + invoke=lambda client, **kwargs: client.actor(ACTOR_ID).start(**kwargs), + method='POST', + path=f'/v2/actors/{ACTOR_ID}/runs', + response_body=RUN_RESPONSE, + tier_secs=30, + ), + id='actor start', + ), + pytest.param( + CallSite( + invoke=lambda client, **kwargs: client.actor(ACTOR_ID).build(version_number='0.0', **kwargs), + method='POST', + path=f'/v2/actors/{ACTOR_ID}/builds', + response_body=BUILD_RESPONSE, + tier_secs=30, + ), + id='actor build', + ), + pytest.param( + CallSite( + invoke=lambda client, **kwargs: client.actor(ACTOR_ID).default_build(**kwargs), + method='GET', + path=f'/v2/actors/{ACTOR_ID}/builds/default', + response_body=BUILD_RESPONSE, + tier_secs=5, + ), + id='actor default build', + ), + pytest.param( + CallSite( + invoke=lambda client, **kwargs: client.task(TASK_ID).start(**kwargs), + method='POST', + path=f'/v2/actor-tasks/{TASK_ID}/runs', + response_body=RUN_RESPONSE, + tier_secs=30, + ), + id='task start', + ), +] + + +@pytest.fixture(params=[pytest.param(ApifyClient, id='sync'), pytest.param(ApifyClientAsync, id='async')]) +def client(request: pytest.FixtureRequest, httpserver: HTTPServer) -> ApifyClient | ApifyClientAsync: + return request.param( + token='test', + api_url=httpserver.url_for('/').removesuffix('/'), + timeout_short=timedelta(seconds=5), + timeout_medium=timedelta(seconds=30), + ) + + +async def call_and_capture_timeout( + client: ApifyClient | ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, + httpserver: HTTPServer, + call_site: CallSite, + **kwargs: Any, +) -> float | None: + """Invoke the call site against a mocked endpoint and return the timeout of the first request attempt.""" + response = Response(json.dumps(call_site.response_body), status=200, mimetype='application/json') + httpserver.expect_request(call_site.path, method=call_site.method).respond_with_response(response) + http_client = client.http_client + send_request = Mock(wraps=http_client.send_request) + monkeypatch.setattr(http_client, 'send_request', send_request) + + result = call_site.invoke(client, **kwargs) + if inspect.isawaitable(result): + await result + + return send_request.call_args_list[0].kwargs['timeout'] + + +@pytest.mark.parametrize('call_site', CALL_SITES) +async def test_default_timeout_covers_wait_for_finish( + client: ApifyClient | ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, + httpserver: HTTPServer, + call_site: CallSite, +) -> None: + """The default timeout tier is extended by `wait_for_finish`, so the request outlasts the API's hold.""" + timeout = await call_and_capture_timeout(client, monkeypatch, httpserver, call_site, wait_for_finish=60) + + assert timeout == call_site.tier_secs + 60 + + +@pytest.mark.parametrize('call_site', CALL_SITES) +async def test_explicit_timeout_ignores_wait_for_finish( + client: ApifyClient | ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, + httpserver: HTTPServer, + call_site: CallSite, +) -> None: + """An explicit `timedelta` timeout is used as given, whatever `wait_for_finish` is.""" + timeout = await call_and_capture_timeout( + client, monkeypatch, httpserver, call_site, wait_for_finish=60, timeout=timedelta(seconds=7) + ) + + assert timeout == 7 + + +@pytest.mark.parametrize( + ('timeout', 'wait_for_finish', 'expected'), + [ + pytest.param('medium', None, 'medium', id='no wait'), + pytest.param('medium', 0, 'medium', id='zero wait'), + pytest.param('medium', -5, 'medium', id='negative wait'), + pytest.param('medium', 20, timedelta(seconds=50), id='medium tier'), + pytest.param('short', 20, timedelta(seconds=25), id='short tier'), + pytest.param('medium', 600, timedelta(seconds=90), id='wait above API maximum'), + pytest.param('no_timeout', 60, 'no_timeout', id='no timeout'), + pytest.param(timedelta(seconds=7), 60, timedelta(seconds=7), id='explicit timedelta'), + ], +) +def test_timeout_for_wait_for_finish( + client: ApifyClient | ApifyClientAsync, + timeout: Timeout, + wait_for_finish: int | None, + expected: Timeout, +) -> None: + """A timeout tier is extended by `wait_for_finish`, capped at the API's 60 s hold; other timeouts stay as given.""" + actor_client = client.actor(ACTOR_ID) + + assert actor_client._timeout_for_wait_for_finish(timeout, wait_for_finish) == expected From e49710033ffa8518409345742029cf27a6f5d71f Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 11 Nov 2026 14:14:28 +0100 Subject: [PATCH 2/3] docs: describe the `wait_for_finish` timeout extension --- docs/02_concepts/11_timeouts.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/02_concepts/11_timeouts.mdx b/docs/02_concepts/11_timeouts.mdx index 3bd0748e..5adccdf4 100644 --- a/docs/02_concepts/11_timeouts.mdx +++ b/docs/02_concepts/11_timeouts.mdx @@ -59,6 +59,10 @@ client.dataset('id').list_items(timeout='no_timeout') A `timedelta` longer than `timeout_max` is capped at `timeout_max`, and the client logs a warning. To let such a call use its full timeout, raise `timeout_max` in the client constructor. +## Waiting for a job to finish + +Methods with a `wait_for_finish` parameter, such as `ActorClient.start` and `ActorClient.build`, ask the API to hold the response until the run or build finishes, for at most 60 seconds. When `timeout` is a tier, the client adds `wait_for_finish` (capped at 60 seconds) to the tier's value, so the request doesn't time out while the API still holds it. A `timedelta` and `'no_timeout'` are used as given. + ## Interaction with retries Timeouts work together with the [retry system](/api/client/python/docs/concepts/retries). When a request times out, it counts as a failed attempt and triggers a retry (up to `max_retries`). The timeout applies to each individual attempt, not the total time across all retries. From 4f5ec75958c45f2606d8f5f4c7a0254e732531da Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 11 Nov 2026 14:14:31 +0100 Subject: [PATCH 3/3] refactor: split the guard in `_timeout_for_wait_for_finish` --- src/apify_client/_resource_clients/_resource_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/apify_client/_resource_clients/_resource_client.py b/src/apify_client/_resource_clients/_resource_client.py index 77abeb57..d9398f7a 100644 --- a/src/apify_client/_resource_clients/_resource_client.py +++ b/src/apify_client/_resource_clients/_resource_client.py @@ -161,7 +161,9 @@ def _timeout_for_wait_for_finish(self, timeout: Timeout, wait_for_finish: int | A request that times out while the API still holds it is retried, and a retried POST starts a second run or build. An explicit `timedelta` and `'no_timeout'` are used as given. """ - if isinstance(timeout, timedelta) or timeout == 'no_timeout' or wait_for_finish is None or wait_for_finish <= 0: + if isinstance(timeout, timedelta) or timeout == 'no_timeout': + return timeout + if wait_for_finish is None or wait_for_finish <= 0: return timeout hold = min(timedelta(seconds=wait_for_finish), MAX_WAIT_FOR_FINISH_HOLD) return self._http_client._resolve_timeout(timeout) + hold # noqa: SLF001