Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/02_concepts/11_timeouts.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ client.dataset('id').list_items(timeout='no_timeout')

A `timedelta` longer than `timeout_max` is capped at `timeout_max`, and the client logs a warning. To let such a call use its full timeout, raise `timeout_max` in the client constructor.

## Waiting for a job to finish

Methods with a `wait_for_finish` parameter, such as <ApiLink to="class/ActorClient#start">`ActorClient.start`</ApiLink> and <ApiLink to="class/ActorClient#build">`ActorClient.build`</ApiLink>, ask the API to hold the response until the run or build finishes, for at most 60 seconds. When `timeout` is a tier, the client adds `wait_for_finish` (capped at 60 seconds) to the tier's value, so the request doesn't time out while the API still holds it. A `timedelta` and `'no_timeout'` are used as given.

## Interaction with retries

Timeouts work together with the [retry system](/api/client/python/docs/concepts/retries). When a request times out, it counts as a failed attempt and triggers a retry (up to `max_retries`). The timeout applies to each individual attempt, not the total time across all retries.
3 changes: 3 additions & 0 deletions src/apify_client/_consts.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
DEFAULT_WAIT_WHEN_JOB_NOT_EXIST = timedelta(seconds=3)
"""How long to wait for a job to exist before giving up."""

MAX_WAIT_FOR_FINISH_HOLD = timedelta(seconds=60)
"""Longest time the API holds a response for the `waitForFinish` parameter."""

OVERRIDABLE_DEFAULT_HEADERS = {'Accept', 'Authorization', 'Accept-Encoding', 'User-Agent'}
"""Headers that can be overridden by users, but will trigger a warning if they do so, as it may lead to API errors."""

Expand Down
19 changes: 18 additions & 1 deletion src/apify_client/_resource_clients/_resource_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@
from typing import TYPE_CHECKING, Any, Literal, get_args
from urllib.parse import urlencode, urlparse, urlunparse

from apify_client._consts import DEFAULT_WAIT_FOR_FINISH, DEFAULT_WAIT_WHEN_JOB_NOT_EXIST
from apify_client._consts import (
DEFAULT_WAIT_FOR_FINISH,
DEFAULT_WAIT_WHEN_JOB_NOT_EXIST,
MAX_WAIT_FOR_FINISH_HOLD,
)
from apify_client._docs import docs_group
from apify_client._logging import WithLogDetailsClient
from apify_client._utils.errors import catch_not_found_for_resource_or_throw, catch_not_found_or_throw
Expand Down Expand Up @@ -151,6 +155,19 @@ def _build_params(self, **kwargs: QueryParamValue) -> dict[str, QueryParamValue]
merged = {**self._default_params, **kwargs}
return {k: v for k, v in merged.items() if v is not None}

def _timeout_for_wait_for_finish(self, timeout: Timeout, wait_for_finish: int | None) -> Timeout:
"""Extend a timeout tier by the time the API may hold the response for `wait_for_finish`.

A request that times out while the API still holds it is retried, and a retried POST starts a second run or
build. An explicit `timedelta` and `'no_timeout'` are used as given.
"""
if isinstance(timeout, timedelta) or timeout == 'no_timeout':
return timeout
if wait_for_finish is None or wait_for_finish <= 0:
return timeout
hold = min(timedelta(seconds=wait_for_finish), MAX_WAIT_FOR_FINISH_HOLD)
return self._http_client._resolve_timeout(timeout) + hold # noqa: SLF001

@staticmethod
def _clean_json_payload(data: dict) -> dict:
"""Remove unset top-level fields from an API request payload.
Expand Down
30 changes: 18 additions & 12 deletions src/apify_client/_resource_clients/actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,8 @@ def start(
than the whole memory limit of the account is rejected with `actor-memory-limit-exceeded` as well and
never starts, so `True` retries it forever. A streamed `run_input` that cannot be rewound, such as a
generator, is sent only once, so its start is not retried.
timeout: Timeout for the API HTTP request.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The run object.
Expand All @@ -360,7 +361,7 @@ def start(
headers={'content-type': content_type},
data=run_input,
params=request_params,
timeout=timeout,
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish),
),
wait_for_resources=wait_for_resources,
)
Expand Down Expand Up @@ -497,7 +498,8 @@ def build(
This is to enable quick rebuild during development. By default, the cache is not used.
wait_for_finish: The maximum number of seconds the server waits for the build to finish before returning.
By default it is 0, the maximum value is 60.
timeout: Timeout for the API HTTP request.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The build object.
Expand All @@ -514,7 +516,7 @@ def build(
url=self._build_url('builds'),
method='POST',
params=request_params,
timeout=timeout,
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish),
)

result = response_to_dict(response)
Expand Down Expand Up @@ -547,7 +549,8 @@ def default_build(
Args:
wait_for_finish: The maximum number of seconds the server waits for the build to finish before returning.
By default it is 0, the maximum value is 60.
timeout: Timeout for the API HTTP request.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The resource client for the default build of this Actor.
Expand All @@ -560,7 +563,7 @@ def default_build(
url=self._build_url('builds/default'),
method='GET',
params=request_params,
timeout=timeout,
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish),
)
result = response_to_dict(response)

Expand Down Expand Up @@ -974,7 +977,8 @@ async def start(
than the whole memory limit of the account is rejected with `actor-memory-limit-exceeded` as well and
never starts, so `True` retries it forever. A streamed `run_input` that cannot be rewound, such as a
generator, is sent only once, so its start is not retried.
timeout: Timeout for the API HTTP request.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The run object.
Expand All @@ -1001,7 +1005,7 @@ async def start(
headers={'content-type': content_type},
data=run_input,
params=request_params,
timeout=timeout,
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish),
),
wait_for_resources=wait_for_resources,
)
Expand Down Expand Up @@ -1139,7 +1143,8 @@ async def build(
This is to enable quick rebuild during development. By default, the cache is not used.
wait_for_finish: The maximum number of seconds the server waits for the build to finish before returning.
By default it is 0, the maximum value is 60.
timeout: Timeout for the API HTTP request.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The build object.
Expand All @@ -1156,7 +1161,7 @@ async def build(
url=self._build_url('builds'),
method='POST',
params=request_params,
timeout=timeout,
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish),
)

result = response_to_dict(response)
Expand Down Expand Up @@ -1189,7 +1194,8 @@ async def default_build(
Args:
wait_for_finish: The maximum number of seconds the server waits for the build to finish before returning.
By default it is 0, the maximum value is 60.
timeout: Timeout for the API HTTP request.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The resource client for the default build of this Actor.
Expand All @@ -1202,7 +1208,7 @@ async def default_build(
url=self._build_url('builds/default'),
method='GET',
params=request_params,
timeout=timeout,
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish),
)
result = response_to_dict(response)

Expand Down
10 changes: 6 additions & 4 deletions src/apify_client/_resource_clients/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,8 @@ def start(
`timedelta` stops retrying after that long and raises the last error. A run that requests more memory
than the whole memory limit of the account is rejected with `actor-memory-limit-exceeded` as well and
never starts, so `True` retries it forever.
timeout: Timeout for the API HTTP request.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The run object.
Expand All @@ -313,7 +314,7 @@ def start(
headers={'content-type': 'application/json; charset=utf-8'},
json=task_input.model_dump() if task_input is not None else None,
params=request_params,
timeout=timeout,
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish),
),
wait_for_resources=wait_for_resources,
)
Expand Down Expand Up @@ -724,7 +725,8 @@ async def start(
`timedelta` stops retrying after that long and raises the last error. A run that requests more memory
than the whole memory limit of the account is rejected with `actor-memory-limit-exceeded` as well and
never starts, so `True` retries it forever.
timeout: Timeout for the API HTTP request.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The run object.
Expand All @@ -750,7 +752,7 @@ async def start(
headers={'content-type': 'application/json; charset=utf-8'},
json=task_input.model_dump() if task_input is not None else None,
params=request_params,
timeout=timeout,
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish),
),
wait_for_resources=wait_for_resources,
)
Expand Down
21 changes: 12 additions & 9 deletions src/apify_client/http_clients/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
from importlib import metadata
from typing import TYPE_CHECKING, Any, TypeVar
from typing import TYPE_CHECKING, Any, Literal, TypeVar
from urllib.parse import urlencode

# `Protocol` comes from `typing_extensions`, not `typing`, because its runtime `isinstance` check looks attributes
Expand Down Expand Up @@ -275,6 +275,16 @@ def _is_body_worth_compressing(data: str | bytes | bytearray | StreamedBodySourc
return len(data) >= MIN_COMPRESSION_SIZE
return False

def _resolve_timeout(self, timeout: timedelta | Literal['short', 'medium', 'long']) -> timedelta:
"""Resolve a timeout tier to its configured duration. A `timedelta` is returned as given."""
if timeout == 'short':
return self._timeout_short
if timeout == 'medium':
return self._timeout_medium
if timeout == 'long':
return self._timeout_long
return timeout

def _compute_timeout(self, timeout: Timeout, *, attempt: int) -> int | float | None:
"""Resolve a timeout tier and compute the timeout for a request attempt with exponential increase.

Expand All @@ -292,14 +302,7 @@ def _compute_timeout(self, timeout: Timeout, *, attempt: int) -> int | float | N
if timeout == 'no_timeout':
return None

if timeout == 'short':
resolved = self._timeout_short
elif timeout == 'medium':
resolved = self._timeout_medium
elif timeout == 'long':
resolved = self._timeout_long
else:
resolved = timeout
resolved = self._resolve_timeout(timeout)

if resolved > self._timeout_max:
# Keyed per timeout kind, so retries and repeated calls do not spam the log with the same warning.
Expand Down
Loading
Loading