diff --git a/docs/02_concepts/05_retries.mdx b/docs/02_concepts/05_retries.mdx index 54cf2562..4bf6a7ca 100644 --- a/docs/02_concepts/05_retries.mdx +++ b/docs/02_concepts/05_retries.mdx @@ -12,6 +12,8 @@ import ApiLink from '@theme/ApiLink'; import RetriesAsyncExample from '!!raw-loader!./code/05_retries_async.py'; import RetriesSyncExample from '!!raw-loader!./code/05_retries_sync.py'; +import StatsAsyncExample from '!!raw-loader!./code/05_stats_async.py'; +import StatsSyncExample from '!!raw-loader!./code/05_stats_sync.py'; import WaitForResourcesAsyncExample from '!!raw-loader!./code/05_wait_for_resources_async.py'; import WaitForResourcesSyncExample from '!!raw-loader!./code/05_wait_for_resources_sync.py'; @@ -46,6 +48,29 @@ Retries with exponential backoff help reduce the load on the server and increase +## Request statistics + +The client counts its API calls, HTTP requests, and rate limit errors. To read the counts, use the `stats` property. It returns a `ClientStatistics` object with these fields: + +- `calls` is the number of API calls made through the client. +- `requests` is the number of HTTP requests sent, including retries. +- `rate_limit_errors` maps each attempt to the number of rate limit (429) errors it got. Index `0` is the first attempt, index `1` is the first retry, and so on. Only API calls count here. A 429 from an Actor server called through `ActorServerClient.request` is not counted. + +With a custom HTTP client, `stats` returns the statistics of that client. For details, see [HTTP clients](/api/client/python/docs/concepts/custom-http-clients). + + + + + {StatsAsyncExample} + + + + + {StatsSyncExample} + + + + ## Wait for resources to start a run Starting a run fails with an HTTP 402 error when the account doesn't have enough free memory for the run, or when it already runs as many Actors as its plan allows. The error `type` is `actor-memory-limit-exceeded` or `concurrent-runs-limit-exceeded`. The client doesn't retry these errors on its own, since they clear only after other runs or builds of the account finish. diff --git a/docs/02_concepts/code/05_stats_async.py b/docs/02_concepts/code/05_stats_async.py new file mode 100644 index 00000000..133feb04 --- /dev/null +++ b/docs/02_concepts/code/05_stats_async.py @@ -0,0 +1,13 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + + await apify_client.actor('username/actor-name').call() + + stats = apify_client.stats + print(f'API calls: {stats.calls}, HTTP requests: {stats.requests}') + print(f'Rate limit errors per attempt: {dict(stats.rate_limit_errors)}') diff --git a/docs/02_concepts/code/05_stats_sync.py b/docs/02_concepts/code/05_stats_sync.py new file mode 100644 index 00000000..2e323903 --- /dev/null +++ b/docs/02_concepts/code/05_stats_sync.py @@ -0,0 +1,13 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + + apify_client.actor('username/actor-name').call() + + stats = apify_client.stats + print(f'API calls: {stats.calls}, HTTP requests: {stats.requests}') + print(f'Rate limit errors per attempt: {dict(stats.rate_limit_errors)}') diff --git a/src/apify_client/__init__.py b/src/apify_client/__init__.py index 97534ce3..7efad70e 100644 --- a/src/apify_client/__init__.py +++ b/src/apify_client/__init__.py @@ -1,11 +1,13 @@ from importlib import metadata from ._apify_client import ApifyClient, ApifyClientAsync +from ._statistics import ClientStatistics __version__ = metadata.version('apify-client') __all__ = [ 'ApifyClient', 'ApifyClientAsync', + 'ClientStatistics', '__version__', ] diff --git a/src/apify_client/_apify_client.py b/src/apify_client/_apify_client.py index 255c87ea..808241c9 100644 --- a/src/apify_client/_apify_client.py +++ b/src/apify_client/_apify_client.py @@ -72,7 +72,6 @@ WebhookDispatchCollectionClient, WebhookDispatchCollectionClientAsync, ) -from apify_client._statistics import ClientStatistics from apify_client._utils.http import check_custom_headers, to_api_base_url from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync from apify_client.http_compressors._resolve import resolve_compressor @@ -80,6 +79,7 @@ if TYPE_CHECKING: from datetime import timedelta + from apify_client._statistics import ClientStatistics from apify_client.http_compressors._base import HttpCompressor from apify_client.types import HttpCompressionAlgorithm @@ -182,9 +182,6 @@ def __init__( self._public_base_url = to_api_base_url(api_public_url) """Public base URL for CDN access.""" - self._statistics = ClientStatistics() - """Collector for client request statistics.""" - self._http_client: HttpClient | None = None """HTTP client used to communicate with the Apify API. Lazily initialized on first access.""" @@ -284,6 +281,15 @@ def token(self) -> str | None: """The Apify API token used by the client.""" return self._token + @property + def stats(self) -> ClientStatistics: + """Statistics about API client usage and rate limit errors. + + These are the statistics of the HTTP client in use. A custom HTTP client that overrides `call` counts only + what its own implementation records. + """ + return self.http_client.stats + @property def http_client(self) -> HttpClient: """The HTTP client instance used for API communication. @@ -300,7 +306,6 @@ def http_client(self) -> HttpClient: timeout_max=self._timeout_max, max_retries=self._max_retries, min_delay_between_retries=self._min_delay_between_retries, - statistics=self._statistics, headers=self._headers, user_agent_suffix=self._user_agent_suffix, http_compressor=self._http_compressor, @@ -565,9 +570,6 @@ def __init__( self._public_base_url = to_api_base_url(api_public_url) """Public base URL for CDN access.""" - self._statistics = ClientStatistics() - """Collector for client request statistics.""" - self._http_client: HttpClientAsync | None = None """HTTP client used to communicate with the Apify API. Lazily initialized on first access.""" @@ -667,6 +669,15 @@ def token(self) -> str | None: """The Apify API token used by the client.""" return self._token + @property + def stats(self) -> ClientStatistics: + """Statistics about API client usage and rate limit errors. + + These are the statistics of the HTTP client in use. A custom HTTP client that overrides `call` counts only + what its own implementation records. + """ + return self.http_client.stats + @property def http_client(self) -> HttpClientAsync: """The HTTP client instance used for API communication. @@ -683,7 +694,6 @@ def http_client(self) -> HttpClientAsync: timeout_max=self._timeout_max, max_retries=self._max_retries, min_delay_between_retries=self._min_delay_between_retries, - statistics=self._statistics, headers=self._headers, user_agent_suffix=self._user_agent_suffix, http_compressor=self._http_compressor, diff --git a/src/apify_client/_statistics.py b/src/apify_client/_statistics.py index b1639bb3..18b137db 100644 --- a/src/apify_client/_statistics.py +++ b/src/apify_client/_statistics.py @@ -3,7 +3,10 @@ from collections import defaultdict from dataclasses import dataclass, field +from apify_client._docs import docs_group + +@docs_group('Other') @dataclass class ClientStatistics: """Statistics about API client usage and rate limit errors.""" @@ -15,7 +18,7 @@ class ClientStatistics: """Total number of HTTP requests sent, including retries.""" rate_limit_errors: defaultdict[int, int] = field(default_factory=lambda: defaultdict(int)) - """List tracking which retry attempts encountered rate limit (429) errors.""" + """Number of rate limit (429) errors per attempt, keyed by the 0-based attempt index.""" def add_rate_limit_error(self, attempt: int) -> None: """Add rate limit error for specific attempt. diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 4d478f18..24a6d184 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -190,6 +190,11 @@ def __init__( self._headers = merge_headers(default_headers, headers) + @property + def stats(self) -> ClientStatistics: + """Statistics about API client usage and rate limit errors.""" + return self._statistics + def set_default_authorization(self, token: str) -> None: """Set the `Authorization` header from the token, unless an authorization header is already configured. diff --git a/tests/unit/test_actor_server.py b/tests/unit/test_actor_server.py index f7ba4379..bd455aca 100644 --- a/tests/unit/test_actor_server.py +++ b/tests/unit/test_actor_server.py @@ -495,7 +495,7 @@ def test_request_rate_limit_is_left_out_of_statistics(httpserver: HTTPServer) -> client.actor(_MOCKED_ACTOR_ID).server().request('/flaky') - assert client._statistics.rate_limit_errors == {} + assert client.stats.rate_limit_errors == {} def test_request_returns_last_retried_response(httpserver: HTTPServer) -> None: diff --git a/tests/unit/test_statistics.py b/tests/unit/test_statistics.py index 37634a7b..534b5520 100644 --- a/tests/unit/test_statistics.py +++ b/tests/unit/test_statistics.py @@ -1,8 +1,16 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest +from werkzeug import Response + +import apify_client +from apify_client import ApifyClient, ApifyClientAsync, ClientStatistics +from apify_client.http_clients import ImpitHttpClient, ImpitHttpClientAsync -from apify_client._statistics import ClientStatistics +if TYPE_CHECKING: + from pytest_httpserver import HTTPServer @pytest.mark.parametrize( @@ -120,3 +128,89 @@ def test_add_rate_limit_error_large_attempt() -> None: stats.add_rate_limit_error(10000) assert stats.rate_limit_errors[9999] == 1 + + +def test_client_statistics_is_exported() -> None: + """Test that ClientStatistics is exported from the top-level apify_client package.""" + assert 'ClientStatistics' in apify_client.__all__ + assert apify_client.ClientStatistics is ClientStatistics + + +def test_apify_client_exposes_default_http_client_statistics(httpserver: HTTPServer) -> None: + """Test that ApifyClient.stats tracks calls and requests on the default HTTP client.""" + httpserver.expect_request('/v2/logs/test-id').respond_with_response( + Response('ok', status=200, content_type='text/plain') + ) + client = ApifyClient('test-token', api_url=httpserver.url_for('/')) + + assert isinstance(client.stats, ClientStatistics) + assert client.stats is client.http_client.stats + assert client.stats.calls == 0 + assert client.stats.requests == 0 + + client.log('test-id').get() + + assert client.stats.calls == 1 + assert client.stats.requests == 1 + + +def test_apify_client_exposes_custom_http_client_statistics(httpserver: HTTPServer) -> None: + """Test that ApifyClient.with_custom_http_client points client.stats to the custom HTTP client's stats.""" + httpserver.expect_request('/v2/logs/test-id').respond_with_response( + Response('ok', status=200, content_type='text/plain') + ) + custom_http_client = ImpitHttpClient() + client = ApifyClient.with_custom_http_client( + 'test-token', + api_url=httpserver.url_for('/'), + http_client=custom_http_client, + ) + + assert client.stats is custom_http_client.stats + assert client.stats.calls == 0 + assert client.stats.requests == 0 + + client.log('test-id').get() + + assert client.stats.calls == 1 + assert client.stats.requests == 1 + + +async def test_apify_client_async_exposes_default_http_client_statistics(httpserver: HTTPServer) -> None: + """Test that ApifyClientAsync.stats tracks calls and requests on the default HTTP client.""" + httpserver.expect_request('/v2/logs/test-id').respond_with_response( + Response('ok', status=200, content_type='text/plain') + ) + client = ApifyClientAsync('test-token', api_url=httpserver.url_for('/')) + + assert isinstance(client.stats, ClientStatistics) + assert client.stats is client.http_client.stats + assert client.stats.calls == 0 + assert client.stats.requests == 0 + + await client.log('test-id').get() + + assert client.stats.calls == 1 + assert client.stats.requests == 1 + + +async def test_apify_client_async_exposes_custom_http_client_statistics(httpserver: HTTPServer) -> None: + """Test that ApifyClientAsync.with_custom_http_client points client.stats to the custom HTTP client's stats.""" + httpserver.expect_request('/v2/logs/test-id').respond_with_response( + Response('ok', status=200, content_type='text/plain') + ) + custom_http_client = ImpitHttpClientAsync() + client = ApifyClientAsync.with_custom_http_client( + 'test-token', + api_url=httpserver.url_for('/'), + http_client=custom_http_client, + ) + + assert client.stats is custom_http_client.stats + assert client.stats.calls == 0 + assert client.stats.requests == 0 + + await client.log('test-id').get() + + assert client.stats.calls == 1 + assert client.stats.requests == 1