Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions docs/02_concepts/05_retries.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import ApiLink from '@theme/ApiLink';

import RetriesAsyncExample from '!!raw-loader!./code/05_retries_async.py';
import RetriesSyncExample from '!!raw-loader!./code/05_retries_sync.py';
import StatsAsyncExample from '!!raw-loader!./code/05_stats_async.py';
import StatsSyncExample from '!!raw-loader!./code/05_stats_sync.py';
import WaitForResourcesAsyncExample from '!!raw-loader!./code/05_wait_for_resources_async.py';
import WaitForResourcesSyncExample from '!!raw-loader!./code/05_wait_for_resources_sync.py';

Expand Down Expand Up @@ -46,6 +48,29 @@ Retries with exponential backoff help reduce the load on the server and increase
</TabItem>
</Tabs>

## Request statistics

The client counts its API calls, HTTP requests, and rate limit errors. To read the counts, use the <ApiLink to="class/ApifyClient#stats">`stats`</ApiLink> property. It returns a <ApiLink to="class/ClientStatistics">`ClientStatistics`</ApiLink> object with these fields:

- `calls` is the number of API calls made through the client.
- `requests` is the number of HTTP requests sent, including retries.
- `rate_limit_errors` maps each attempt to the number of rate limit (429) errors it got. Index `0` is the first attempt, index `1` is the first retry, and so on.

With a custom HTTP client, `stats` returns the statistics of that client. For details, see [HTTP clients](/api/client/python/docs/concepts/custom-http-clients).

<Tabs>
<TabItem value="AsyncExample" label="Async client" default>
<CodeBlock className="language-python">
{StatsAsyncExample}
</CodeBlock>
</TabItem>
<TabItem value="SyncExample" label="Sync client">
<CodeBlock className="language-python">
{StatsSyncExample}
</CodeBlock>
</TabItem>
</Tabs>

## Wait for resources to start a run

Starting a run fails with an HTTP 402 error when the account doesn't have enough free memory for the run, or when it already runs as many Actors as its plan allows. The error `type` is `actor-memory-limit-exceeded` or `concurrent-runs-limit-exceeded`. The client doesn't retry these errors on its own, since they clear only after other runs or builds of the account finish.
Expand Down
13 changes: 13 additions & 0 deletions docs/02_concepts/code/05_stats_async.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
from apify_client import ApifyClientAsync

TOKEN = 'MY-APIFY-TOKEN'


async def main() -> None:
apify_client = ApifyClientAsync(TOKEN)

await apify_client.actor('username/actor-name').call()

stats = apify_client.stats
print(f'API calls: {stats.calls}, HTTP requests: {stats.requests}')
print(f'Rate limit errors per attempt: {dict(stats.rate_limit_errors)}')
13 changes: 13 additions & 0 deletions docs/02_concepts/code/05_stats_sync.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
from apify_client import ApifyClient

TOKEN = 'MY-APIFY-TOKEN'


def main() -> None:
apify_client = ApifyClient(TOKEN)

apify_client.actor('username/actor-name').call()

stats = apify_client.stats
print(f'API calls: {stats.calls}, HTTP requests: {stats.requests}')
print(f'Rate limit errors per attempt: {dict(stats.rate_limit_errors)}')
2 changes: 2 additions & 0 deletions src/apify_client/__init__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
from importlib import metadata

from ._apify_client import ApifyClient, ApifyClientAsync
from ._statistics import ClientStatistics

__version__ = metadata.version('apify-client')

__all__ = [
'ApifyClient',
'ApifyClientAsync',
'ClientStatistics',
'__version__',
]
18 changes: 18 additions & 0 deletions src/apify_client/_apify_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,15 @@ def token(self) -> str | None:
"""The Apify API token used by the client."""
return self._token

@property
def stats(self) -> ClientStatistics:
"""Statistics about API client usage and rate limit errors.

These are the statistics of the HTTP client in use. A custom HTTP client that overrides `call` counts only
what its own implementation records.
"""
return self.http_client.stats

@property
def http_client(self) -> HttpClient:
"""The HTTP client instance used for API communication.
Expand Down Expand Up @@ -667,6 +676,15 @@ def token(self) -> str | None:
"""The Apify API token used by the client."""
return self._token

@property
def stats(self) -> ClientStatistics:
"""Statistics about API client usage and rate limit errors.

These are the statistics of the HTTP client in use. A custom HTTP client that overrides `call` counts only
what its own implementation records.
"""
return self.http_client.stats

@property
def http_client(self) -> HttpClientAsync:
"""The HTTP client instance used for API communication.
Expand Down
5 changes: 4 additions & 1 deletion src/apify_client/_statistics.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
from collections import defaultdict
from dataclasses import dataclass, field

from apify_client._docs import docs_group


@docs_group('Other')
@dataclass
class ClientStatistics:
"""Statistics about API client usage and rate limit errors."""
Expand All @@ -15,7 +18,7 @@ class ClientStatistics:
"""Total number of HTTP requests sent, including retries."""

rate_limit_errors: defaultdict[int, int] = field(default_factory=lambda: defaultdict(int))
"""List tracking which retry attempts encountered rate limit (429) errors."""
"""Number of rate limit (429) errors per attempt, keyed by the 0-based attempt index."""

def add_rate_limit_error(self, attempt: int) -> None:
"""Add rate limit error for specific attempt.
Expand Down
5 changes: 5 additions & 0 deletions src/apify_client/http_clients/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,11 @@ def __init__(

self._headers = merge_headers(default_headers, headers)

@property
def stats(self) -> ClientStatistics:
"""Statistics about API client usage and rate limit errors."""
return self._statistics

def set_default_authorization(self, token: str) -> None:
"""Set the `Authorization` header from the token, unless an authorization header is already configured.

Expand Down
96 changes: 95 additions & 1 deletion tests/unit/test_statistics.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,16 @@
from __future__ import annotations

from typing import TYPE_CHECKING

import pytest
from werkzeug import Response

import apify_client
from apify_client import ApifyClient, ApifyClientAsync, ClientStatistics
from apify_client.http_clients import ImpitHttpClient, ImpitHttpClientAsync

from apify_client._statistics import ClientStatistics
if TYPE_CHECKING:
from pytest_httpserver import HTTPServer


@pytest.mark.parametrize(
Expand Down Expand Up @@ -120,3 +128,89 @@ def test_add_rate_limit_error_large_attempt() -> None:

stats.add_rate_limit_error(10000)
assert stats.rate_limit_errors[9999] == 1


def test_client_statistics_is_exported() -> None:
"""Test that ClientStatistics is exported from the top-level apify_client package."""
assert 'ClientStatistics' in apify_client.__all__
assert apify_client.ClientStatistics is ClientStatistics


def test_apify_client_exposes_default_http_client_statistics(httpserver: HTTPServer) -> None:
"""Test that ApifyClient.stats tracks calls and requests on the default HTTP client."""
httpserver.expect_request('/v2/logs/test-id').respond_with_response(
Response('ok', status=200, content_type='text/plain')
)
client = ApifyClient('test-token', api_url=httpserver.url_for('/'))

assert isinstance(client.stats, ClientStatistics)
assert client.stats is client.http_client.stats
assert client.stats.calls == 0
assert client.stats.requests == 0

client.log('test-id').get()

assert client.stats.calls == 1
assert client.stats.requests == 1


def test_apify_client_exposes_custom_http_client_statistics(httpserver: HTTPServer) -> None:
"""Test that ApifyClient.with_custom_http_client points client.stats to the custom HTTP client's stats."""
httpserver.expect_request('/v2/logs/test-id').respond_with_response(
Response('ok', status=200, content_type='text/plain')
)
custom_http_client = ImpitHttpClient()
client = ApifyClient.with_custom_http_client(
'test-token',
api_url=httpserver.url_for('/'),
http_client=custom_http_client,
)

assert client.stats is custom_http_client.stats
assert client.stats.calls == 0
assert client.stats.requests == 0

client.log('test-id').get()

assert client.stats.calls == 1
assert client.stats.requests == 1


async def test_apify_client_async_exposes_default_http_client_statistics(httpserver: HTTPServer) -> None:
"""Test that ApifyClientAsync.stats tracks calls and requests on the default HTTP client."""
httpserver.expect_request('/v2/logs/test-id').respond_with_response(
Response('ok', status=200, content_type='text/plain')
)
client = ApifyClientAsync('test-token', api_url=httpserver.url_for('/'))

assert isinstance(client.stats, ClientStatistics)
assert client.stats is client.http_client.stats
assert client.stats.calls == 0
assert client.stats.requests == 0

await client.log('test-id').get()

assert client.stats.calls == 1
assert client.stats.requests == 1


async def test_apify_client_async_exposes_custom_http_client_statistics(httpserver: HTTPServer) -> None:
"""Test that ApifyClientAsync.with_custom_http_client points client.stats to the custom HTTP client's stats."""
httpserver.expect_request('/v2/logs/test-id').respond_with_response(
Response('ok', status=200, content_type='text/plain')
)
custom_http_client = ImpitHttpClientAsync()
client = ApifyClientAsync.with_custom_http_client(
'test-token',
api_url=httpserver.url_for('/'),
http_client=custom_http_client,
)

assert client.stats is custom_http_client.stats
assert client.stats.calls == 0
assert client.stats.requests == 0

await client.log('test-id').get()

assert client.stats.calls == 1
assert client.stats.requests == 1
Loading