From 810f41403555d31a096c0587e230cc3ba4dc0212 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 3 Aug 2026 09:42:13 +0200 Subject: [PATCH 1/6] perf: Skip request-body compression for already-compressed content types --- docs/02_concepts/13_http_compression.mdx | 23 ++++++++++- src/apify_client/_consts.py | 25 ++++++++++++ src/apify_client/_utils/http.py | 34 +++++++++++++++- src/apify_client/http_clients/_base.py | 11 +++++- tests/unit/test_http_clients.py | 49 ++++++++++++++++++++++++ tests/unit/test_utils.py | 32 +++++++++++++++- 6 files changed, 168 insertions(+), 6 deletions(-) diff --git a/docs/02_concepts/13_http_compression.mdx b/docs/02_concepts/13_http_compression.mdx index 3073ed3c..ce9650da 100644 --- a/docs/02_concepts/13_http_compression.mdx +++ b/docs/02_concepts/13_http_compression.mdx @@ -1,15 +1,34 @@ --- id: http-compression title: HTTP compression -description: The client compresses every request body automatically using gzip by default, with optional brotli via an explicit opt-in. +description: The client compresses request bodies automatically using gzip by default, with optional brotli via an explicit opt-in. --- -The Apify client compresses every request body before sending it to the API. It reduces the amount of data transferred over the network, resulting in faster requests and lower bandwidth usage, especially for large payloads such as Actor inputs, dataset uploads, or key-value store records. +The Apify client compresses request bodies before sending them to the API. It reduces the amount of data transferred over the network, resulting in faster requests and lower bandwidth usage, especially for large payloads such as Actor inputs, dataset uploads, or key-value store records. ## How it works The client compresses request bodies using the compressor configured via the `compression` parameter (default `'gzip'`). The server supports both gzip and brotli and decompresses the request body transparently. +## Already-compressed payloads + +Some payloads carry their own compression, so compressing them again costs CPU and memory while typically making the request slightly larger. The client skips compression when the request's `Content-Type` is one of these: + +- any `image/*`, `audio/*`, or `video/*` type, +- archives such as `application/zip`, `application/gzip`, or `application/x-7z-compressed`, +- web fonts (`font/woff`, `font/woff2`). + +Text-based types are still compressed even under those prefixes, so `image/svg+xml` is compressed as usual. Set an accurate `content_type` when uploading media to a key-value store to benefit from this: + +```python +kvs = client.key_value_store('MY-STORE-ID') + +with open('screenshot.png', 'rb') as file: + kvs.set_record('screenshot', file, content_type='image/png') +``` + +Without an explicit content type the record is sent as `application/octet-stream`, which the client cannot tell apart from uncompressed binary data and therefore still compresses. + ## Configuration To choose the compression algorithm, pass `compression` to the client constructor: diff --git a/src/apify_client/_consts.py b/src/apify_client/_consts.py index 134e0e7b..2abd18a7 100644 --- a/src/apify_client/_consts.py +++ b/src/apify_client/_consts.py @@ -34,3 +34,28 @@ OVERRIDABLE_DEFAULT_HEADERS = {'Accept', 'Authorization', 'Accept-Encoding', 'User-Agent'} """Headers that can be overridden by users, but will trigger a warning if they do so, as it may lead to API errors.""" + +ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ('audio/', 'image/', 'video/') +"""Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work.""" + +ALREADY_COMPRESSED_MEDIA_TYPES = frozenset( + { + 'application/gzip', + 'application/java-archive', + 'application/vnd.rar', + 'application/x-7z-compressed', + 'application/x-bzip', + 'application/x-bzip2', + 'application/x-gzip', + 'application/x-rar-compressed', + 'application/x-xz', + 'application/zip', + 'application/zstd', + 'font/woff', + 'font/woff2', + } +) +"""Exact media types whose payloads carry their own compression.""" + +COMPRESSIBLE_MEDIA_TYPE_SUFFIXES = ('+json', '+xml') +"""Structured syntax suffixes marking a media type as text even under an already-compressed prefix (`image/svg+xml`).""" diff --git a/src/apify_client/_utils/http.py b/src/apify_client/_utils/http.py index 448746bd..5bfb3dc4 100644 --- a/src/apify_client/_utils/http.py +++ b/src/apify_client/_utils/http.py @@ -3,7 +3,12 @@ import warnings from typing import TYPE_CHECKING -from apify_client._consts import OVERRIDABLE_DEFAULT_HEADERS +from apify_client._consts import ( + ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES, + ALREADY_COMPRESSED_MEDIA_TYPES, + COMPRESSIBLE_MEDIA_TYPE_SUFFIXES, + OVERRIDABLE_DEFAULT_HEADERS, +) if TYPE_CHECKING: from apify_client.http_clients import HttpResponse @@ -21,6 +26,33 @@ def to_safe_id(id: str) -> str: return id.replace('/', '~') +def is_compressible_content_type(content_type: str | None) -> bool: + """Decide whether a request body with the given content type is worth compressing. + + Images, audio, video and archives already carry their own compression. Running them through gzip or brotli + burns CPU, holds a second full copy of the body in memory, and usually produces output slightly larger than + the input. A body with no content type is assumed to be compressible. + + Args: + content_type: The value of the `Content-Type` header, if any. + + Returns: + `True` if the body should be compressed before it is sent. + """ + if not content_type: + return True + + # `Content-Type` is case-insensitive and may carry parameters, for example `text/plain; charset=utf-8`. + media_type = content_type.split(';', 1)[0].strip().lower() + + if media_type.endswith(COMPRESSIBLE_MEDIA_TYPE_SUFFIXES): + return True + + return not ( + media_type in ALREADY_COMPRESSED_MEDIA_TYPES or media_type.startswith(ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES) + ) + + def response_to_dict(response: HttpResponse) -> dict: """Parse the API response as a dictionary and validate its type. diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 7645228f..3c23c31b 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -19,6 +19,7 @@ ) from apify_client._docs import docs_group from apify_client._statistics import ClientStatistics +from apify_client._utils.http import is_compressible_content_type from apify_client._utils.time import to_seconds from apify_client.http_compressors._gzip import GzipHttpCompressor @@ -253,8 +254,14 @@ def _prepare_request_call( data = data.encode('utf-8') elif isinstance(data, bytearray): data = bytes(data) - data = self._http_compressor.compress(data) - headers = self._merge_headers(headers, {'Content-Encoding': self._http_compressor.content_encoding}) + + content_type = next((value for key, value in headers.items() if key.lower() == 'content-type'), None) + if is_compressible_content_type(content_type): + data = self._http_compressor.compress(data) + headers = self._merge_headers(headers, {'Content-Encoding': self._http_compressor.content_encoding}) + else: + # The body goes out as-is, so any caller-supplied encoding would misdescribe it. + headers = {key: value for key, value in headers.items() if key.lower() != 'content-encoding'} return (headers, self._parse_params(params), data) diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index 317680cb..a81d4a88 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -461,6 +461,55 @@ def test_prepare_request_call_with_bytearray_data(compressor_case: tuple) -> Non assert decompress(data) == b'test bytearray' +@pytest.mark.parametrize( + 'content_type', + [ + pytest.param('image/png', id='image'), + pytest.param('video/mp4', id='video'), + pytest.param('application/zip', id='archive'), + ], +) +def test_prepare_request_call_skips_compression_for_already_compressed_content(content_type: str) -> None: + """An already-compressed body is sent verbatim and carries no `Content-Encoding` header.""" + client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) + + headers, _params, data = client._prepare_request_call( + headers={'content-type': content_type}, + data=b'\x89PNG binary', + ) + + assert data == b'\x89PNG binary' + assert not any(key.lower() == 'content-encoding' for key in headers) + + +def test_prepare_request_call_drops_caller_content_encoding_when_compression_is_skipped() -> None: + """Skipping compression also strips a caller-supplied `Content-Encoding`, which would misdescribe the body.""" + client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) + + headers, _params, data = client._prepare_request_call( + headers={'content-type': 'image/jpeg', 'content-encoding': 'br'}, + data=b'jpeg binary', + ) + + assert data == b'jpeg binary' + assert not any(key.lower() == 'content-encoding' for key in headers) + + +def test_prepare_request_call_compresses_text_content_types(compressor_case: tuple) -> None: + """A text content type is still compressed, even when it sits under an already-compressed prefix.""" + compressor, content_encoding, decompress = compressor_case + client = _ConcreteHttpClient(http_compressor=compressor) + + headers, _params, data = client._prepare_request_call( + headers={'content-type': 'image/svg+xml'}, + data=b'', + ) + + assert headers['Content-Encoding'] == content_encoding + assert isinstance(data, bytes) + assert decompress(data) == b'' + + def test_prepare_request_call_json_and_data_error() -> None: """Test _prepare_request_call raises error when both json and data are provided.""" client = _ConcreteHttpClient() diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a52d3fb4..2e3ed91f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -16,7 +16,12 @@ from apify_client._utils.crypto import create_hmac_signature, create_storage_content_signature, encode_base62 from apify_client._utils.encoding import encode_key_value_store_record_value, encode_webhooks_to_base64 from apify_client._utils.errors import catch_not_found_or_throw, is_retryable_error -from apify_client._utils.http import response_to_dict, response_to_list, to_safe_id +from apify_client._utils.http import ( + is_compressible_content_type, + response_to_dict, + response_to_list, + to_safe_id, +) from apify_client.errors import ApifyApiError, InvalidResponseBodyError if TYPE_CHECKING: @@ -256,6 +261,31 @@ def test_encode_key_value_store_record_value_bytesio() -> None: assert content_type == 'application/octet-stream' +@pytest.mark.parametrize( + ('content_type', 'expected'), + [ + pytest.param(None, True, id='missing'), + pytest.param('', True, id='empty'), + pytest.param('application/json', True, id='json'), + pytest.param('text/plain; charset=utf-8', True, id='text with parameters'), + pytest.param('application/octet-stream', True, id='unknown binary'), + pytest.param('application/vnd.api+json', True, id='structured json suffix'), + pytest.param('image/svg+xml', True, id='svg under a compressed prefix'), + pytest.param('IMAGE/SVG+XML; charset=utf-8', True, id='svg uppercase with parameters'), + pytest.param('image/png', False, id='image prefix'), + pytest.param('video/mp4', False, id='video prefix'), + pytest.param('audio/mpeg', False, id='audio prefix'), + pytest.param('application/zip', False, id='archive'), + pytest.param('application/x-gzip', False, id='gzip archive'), + pytest.param('font/woff2', False, id='web font'), + pytest.param(' Image/PNG ', False, id='surrounding whitespace and mixed case'), + ], +) +def test_is_compressible_content_type(content_type: str | None, *, expected: bool) -> None: + """Already-compressed media types are reported as not worth compressing, everything else as compressible.""" + assert is_compressible_content_type(content_type) is expected + + def test_response_to_dict() -> None: """Test parsing response as dictionary.""" mock_response = Mock() From d88cf4ec60fa9023e5140b6b50fe18811be0b3b7 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 3 Aug 2026 09:50:58 +0200 Subject: [PATCH 2/6] docs: Move the skip-compression example into dedicated code files --- docs/02_concepts/13_http_compression.mdx | 25 ++++++++++++++----- .../code/13_skip_compression_async.py | 16 ++++++++++++ .../code/13_skip_compression_sync.py | 15 +++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 docs/02_concepts/code/13_skip_compression_async.py create mode 100644 docs/02_concepts/code/13_skip_compression_sync.py diff --git a/docs/02_concepts/13_http_compression.mdx b/docs/02_concepts/13_http_compression.mdx index ce9650da..b1b4291c 100644 --- a/docs/02_concepts/13_http_compression.mdx +++ b/docs/02_concepts/13_http_compression.mdx @@ -4,6 +4,13 @@ title: HTTP compression description: The client compresses request bodies automatically using gzip by default, with optional brotli via an explicit opt-in. --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import SkipCompressionAsyncExample from '!!raw-loader!./code/13_skip_compression_async.py'; +import SkipCompressionSyncExample from '!!raw-loader!./code/13_skip_compression_sync.py'; + The Apify client compresses request bodies before sending them to the API. It reduces the amount of data transferred over the network, resulting in faster requests and lower bandwidth usage, especially for large payloads such as Actor inputs, dataset uploads, or key-value store records. ## How it works @@ -20,12 +27,18 @@ Some payloads carry their own compression, so compressing them again costs CPU a Text-based types are still compressed even under those prefixes, so `image/svg+xml` is compressed as usual. Set an accurate `content_type` when uploading media to a key-value store to benefit from this: -```python -kvs = client.key_value_store('MY-STORE-ID') - -with open('screenshot.png', 'rb') as file: - kvs.set_record('screenshot', file, content_type='image/png') -``` + + + + {SkipCompressionAsyncExample} + + + + + {SkipCompressionSyncExample} + + + Without an explicit content type the record is sent as `application/octet-stream`, which the client cannot tell apart from uncompressed binary data and therefore still compresses. diff --git a/docs/02_concepts/code/13_skip_compression_async.py b/docs/02_concepts/code/13_skip_compression_async.py new file mode 100644 index 00000000..ce7847f5 --- /dev/null +++ b/docs/02_concepts/code/13_skip_compression_async.py @@ -0,0 +1,16 @@ +import asyncio +from pathlib import Path + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + kvs_client = apify_client.key_value_store('MY-KVS-ID') + + screenshot = await asyncio.to_thread(Path('screenshot.png').read_bytes) + + # The explicit content type lets the client skip compressing the PNG. + await kvs_client.set_record('screenshot', screenshot, content_type='image/png') diff --git a/docs/02_concepts/code/13_skip_compression_sync.py b/docs/02_concepts/code/13_skip_compression_sync.py new file mode 100644 index 00000000..eeb00668 --- /dev/null +++ b/docs/02_concepts/code/13_skip_compression_sync.py @@ -0,0 +1,15 @@ +from pathlib import Path + +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + kvs_client = apify_client.key_value_store('MY-KVS-ID') + + screenshot = Path('screenshot.png').read_bytes() + + # The explicit content type lets the client skip compressing the PNG. + kvs_client.set_record('screenshot', screenshot, content_type='image/png') From e651bac78744684b54a4ef0ba345d76c9df77dfd Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 3 Aug 2026 10:11:41 +0200 Subject: [PATCH 3/6] fix: Compress raw media types that sit under an already-compressed prefix --- src/apify_client/_consts.py | 26 ++++++++++++++++++++++++++ src/apify_client/_utils/http.py | 6 ++++-- tests/unit/test_utils.py | 10 ++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/apify_client/_consts.py b/src/apify_client/_consts.py index 2abd18a7..b7775e07 100644 --- a/src/apify_client/_consts.py +++ b/src/apify_client/_consts.py @@ -40,8 +40,13 @@ ALREADY_COMPRESSED_MEDIA_TYPES = frozenset( { + 'application/epub+zip', 'application/gzip', 'application/java-archive', + 'application/vnd.android.package-archive', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.rar', 'application/x-7z-compressed', 'application/x-bzip', @@ -49,6 +54,7 @@ 'application/x-gzip', 'application/x-rar-compressed', 'application/x-xz', + 'application/x-zip-compressed', 'application/zip', 'application/zstd', 'font/woff', @@ -57,5 +63,25 @@ ) """Exact media types whose payloads carry their own compression.""" +COMPRESSIBLE_MEDIA_TYPES = frozenset( + { + 'audio/aiff', + 'audio/basic', + 'audio/l16', + 'audio/vnd.wave', + 'audio/wav', + 'audio/wave', + 'audio/x-aiff', + 'audio/x-wav', + 'image/bmp', + 'image/tiff', + 'image/vnd.adobe.photoshop', + 'image/vnd.microsoft.icon', + 'image/x-icon', + 'image/x-ms-bmp', + } +) +"""Uncompressed media types that sit under an already-compressed prefix, so compressing them still pays off.""" + COMPRESSIBLE_MEDIA_TYPE_SUFFIXES = ('+json', '+xml') """Structured syntax suffixes marking a media type as text even under an already-compressed prefix (`image/svg+xml`).""" diff --git a/src/apify_client/_utils/http.py b/src/apify_client/_utils/http.py index 5bfb3dc4..8d137ae3 100644 --- a/src/apify_client/_utils/http.py +++ b/src/apify_client/_utils/http.py @@ -7,6 +7,7 @@ ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES, ALREADY_COMPRESSED_MEDIA_TYPES, COMPRESSIBLE_MEDIA_TYPE_SUFFIXES, + COMPRESSIBLE_MEDIA_TYPES, OVERRIDABLE_DEFAULT_HEADERS, ) @@ -31,7 +32,8 @@ def is_compressible_content_type(content_type: str | None) -> bool: Images, audio, video and archives already carry their own compression. Running them through gzip or brotli burns CPU, holds a second full copy of the body in memory, and usually produces output slightly larger than - the input. A body with no content type is assumed to be compressible. + the input. Formats that are raw despite such a media type, for example `image/bmp` or `audio/wav`, are still + compressed. A body with no content type is assumed to be compressible. Args: content_type: The value of the `Content-Type` header, if any. @@ -45,7 +47,7 @@ def is_compressible_content_type(content_type: str | None) -> bool: # `Content-Type` is case-insensitive and may carry parameters, for example `text/plain; charset=utf-8`. media_type = content_type.split(';', 1)[0].strip().lower() - if media_type.endswith(COMPRESSIBLE_MEDIA_TYPE_SUFFIXES): + if media_type in COMPRESSIBLE_MEDIA_TYPES or media_type.endswith(COMPRESSIBLE_MEDIA_TYPE_SUFFIXES): return True return not ( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 2e3ed91f..fbca63b6 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -272,11 +272,21 @@ def test_encode_key_value_store_record_value_bytesio() -> None: pytest.param('application/vnd.api+json', True, id='structured json suffix'), pytest.param('image/svg+xml', True, id='svg under a compressed prefix'), pytest.param('IMAGE/SVG+XML; charset=utf-8', True, id='svg uppercase with parameters'), + pytest.param('image/bmp', True, id='raw bitmap under a compressed prefix'), + pytest.param('image/tiff', True, id='tiff under a compressed prefix'), + pytest.param('audio/wav', True, id='raw audio under a compressed prefix'), pytest.param('image/png', False, id='image prefix'), pytest.param('video/mp4', False, id='video prefix'), pytest.param('audio/mpeg', False, id='audio prefix'), pytest.param('application/zip', False, id='archive'), pytest.param('application/x-gzip', False, id='gzip archive'), + pytest.param('application/x-zip-compressed', False, id='windows zip archive'), + pytest.param('application/epub+zip', False, id='zip container with a suffix'), + pytest.param( + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + False, + id='office open xml document', + ), pytest.param('font/woff2', False, id='web font'), pytest.param(' Image/PNG ', False, id='surrounding whitespace and mixed case'), ], From 286f68b37022a8fe432cb59ccd499c0b4353335b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 3 Aug 2026 10:11:48 +0200 Subject: [PATCH 4/6] fix: Drop caller Content-Encoding whenever the request body is not compressed --- src/apify_client/http_clients/_base.py | 22 +++++++----- src/apify_client/http_clients/_impit.py | 4 +-- tests/unit/test_http_clients.py | 47 ++++++++++++++++++++----- 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 3c23c31b..3862aa68 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -233,22 +233,25 @@ def _prepare_request_call( ) -> tuple[dict[str, str], dict[str, Any] | None, bytes | None]: """Prepare headers, params, and body for an HTTP request. - Merges the client's default headers (including authorization) with per-request headers, - serializes JSON and compresses the body. Header names are treated case-insensitively and - per-request values win over the client defaults. For JSON bodies, a `Content-Type` header - is set unless the caller supplied one. + Merges the client's default headers (including authorization) with per-request headers, serializes JSON + and compresses the body unless its content type says the payload is already compressed. Header names are + treated case-insensitively and per-request values win over the client defaults. For JSON bodies, a + `Content-Type` header is set unless the caller supplied one. `Content-Encoding` always describes what was + actually applied to the body, so a caller-supplied value is dropped whenever nothing was compressed. """ if json is not None and data is not None: raise ValueError('Cannot pass both "json" and "data" parameters at the same time!') headers = self._merge_headers(self._headers, headers) - # Dump JSON data to string so it can be compressed. + # Dump JSON data to a string so it can be sent as a request body. if json is not None: data = jsonlib.dumps(json, ensure_ascii=False, allow_nan=False, default=str).encode('utf-8') if not any(key.lower() == 'content-type' for key in headers): headers['Content-Type'] = 'application/json' + compressed = False + if isinstance(data, (str, bytes, bytearray)): if isinstance(data, str): data = data.encode('utf-8') @@ -259,9 +262,12 @@ def _prepare_request_call( if is_compressible_content_type(content_type): data = self._http_compressor.compress(data) headers = self._merge_headers(headers, {'Content-Encoding': self._http_compressor.content_encoding}) - else: - # The body goes out as-is, so any caller-supplied encoding would misdescribe it. - headers = {key: value for key, value in headers.items() if key.lower() != 'content-encoding'} + compressed = True + + # Anything left uncompressed goes out as-is - a file-like body included - so a caller-supplied encoding + # would misdescribe it. + if data is not None and not compressed: + headers = {key: value for key, value in headers.items() if key.lower() != 'content-encoding'} return (headers, self._parse_params(params), data) diff --git a/src/apify_client/http_clients/_impit.py b/src/apify_client/http_clients/_impit.py index c3ef7212..a082e962 100644 --- a/src/apify_client/http_clients/_impit.py +++ b/src/apify_client/http_clients/_impit.py @@ -395,8 +395,8 @@ async def call( self._statistics.calls += 1 # Serializing and compressing a request body is CPU-bound and would block the event loop, so - # offload request preparation to a worker thread whenever there is a body to compress. Bodyless - # requests skip the thread hop, as they have no expensive work to move off the loop. + # offload request preparation to a worker thread whenever there is a body. Bodyless requests + # skip the thread hop, as they have no expensive work to move off the loop. if json is not None or data is not None: prepared_headers, prepared_params, content = await asyncio.to_thread( self._prepare_request_call, diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index a81d4a88..6755728a 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -2,10 +2,11 @@ import asyncio import gzip +import io import threading import time from datetime import UTC, datetime, timedelta -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from unittest.mock import AsyncMock, Mock import brotli @@ -470,8 +471,8 @@ def test_prepare_request_call_with_bytearray_data(compressor_case: tuple) -> Non ], ) def test_prepare_request_call_skips_compression_for_already_compressed_content(content_type: str) -> None: - """An already-compressed body is sent verbatim and carries no `Content-Encoding` header.""" - client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) + """An already-compressed body is sent verbatim, carries no `Content-Encoding`, and keeps every other header.""" + client = _ConcreteHttpClient(token='test_token', http_compressor=GzipHttpCompressor()) headers, _params, data = client._prepare_request_call( headers={'content-type': content_type}, @@ -480,6 +481,9 @@ def test_prepare_request_call_skips_compression_for_already_compressed_content(c assert data == b'\x89PNG binary' assert not any(key.lower() == 'content-encoding' for key in headers) + assert headers['Authorization'] == 'Bearer test_token' + assert headers['content-type'] == content_type + assert headers['User-Agent'] == client._headers['User-Agent'] def test_prepare_request_call_drops_caller_content_encoding_when_compression_is_skipped() -> None: @@ -495,19 +499,44 @@ def test_prepare_request_call_drops_caller_content_encoding_when_compression_is_ assert not any(key.lower() == 'content-encoding' for key in headers) -def test_prepare_request_call_compresses_text_content_types(compressor_case: tuple) -> None: - """A text content type is still compressed, even when it sits under an already-compressed prefix.""" +def test_prepare_request_call_drops_caller_content_encoding_for_a_streamed_body() -> None: + """A body that is streamed rather than compressed, such as a file-like object, also loses `Content-Encoding`.""" + client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) + stream = io.BytesIO(b'raw payload') + + headers, _params, data = client._prepare_request_call( + headers={'content-encoding': 'gzip'}, + data=cast('bytes', stream), + ) + + assert data is stream + assert not any(key.lower() == 'content-encoding' for key in headers) + + +@pytest.mark.parametrize( + 'content_type', + [ + pytest.param('image/svg+xml', id='structured xml suffix'), + pytest.param('image/bmp', id='raw bitmap'), + pytest.param('audio/wav', id='raw audio'), + ], +) +def test_prepare_request_call_compresses_exceptions_to_compressed_prefixes( + content_type: str, + compressor_case: tuple, +) -> None: + """Types that are text or raw are compressed even when they sit under an already-compressed prefix.""" compressor, content_encoding, decompress = compressor_case client = _ConcreteHttpClient(http_compressor=compressor) headers, _params, data = client._prepare_request_call( - headers={'content-type': 'image/svg+xml'}, - data=b'', + headers={'content-type': content_type}, + data=b'raw payload', ) assert headers['Content-Encoding'] == content_encoding assert isinstance(data, bytes) - assert decompress(data) == b'' + assert decompress(data) == b'raw payload' def test_prepare_request_call_json_and_data_error() -> None: @@ -569,7 +598,7 @@ def test_prepare_request_call_json_keeps_caller_content_type() -> None: def test_prepare_request_call_replaces_caller_content_encoding() -> None: - """The Content-Encoding header always reflects the compressor actually applied, replacing any caller value.""" + """A compressed body reports the compressor actually applied, replacing any caller-supplied Content-Encoding.""" client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) headers, _params, _data = client._prepare_request_call(headers={'content-encoding': 'br'}, data='payload') From 928a1608f208eae49c6b3784090009eeb3f8120f Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 3 Aug 2026 10:11:56 +0200 Subject: [PATCH 5/6] docs: Clarify which payloads skip request-body compression --- docs/02_concepts/13_http_compression.mdx | 13 +++++++------ docs/02_concepts/code/13_skip_compression_async.py | 4 ++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/02_concepts/13_http_compression.mdx b/docs/02_concepts/13_http_compression.mdx index b1b4291c..f5fa9609 100644 --- a/docs/02_concepts/13_http_compression.mdx +++ b/docs/02_concepts/13_http_compression.mdx @@ -19,13 +19,14 @@ The client compresses request bodies using the compressor configured via the `co ## Already-compressed payloads -Some payloads carry their own compression, so compressing them again costs CPU and memory while typically making the request slightly larger. The client skips compression when the request's `Content-Type` is one of these: +Some payloads carry their own compression, so compressing them again costs CPU and memory while making the request slightly larger. The client skips compression when the request's `Content-Type` is one of these: -- any `image/*`, `audio/*`, or `video/*` type, -- archives such as `application/zip`, `application/gzip`, or `application/x-7z-compressed`, -- web fonts (`font/woff`, `font/woff2`). +- any `image/*`, `audio/*`, or `video/*` type +- archives such as `application/zip`, `application/gzip`, or `application/x-7z-compressed` +- office documents and packages built on ZIP, such as `.docx`, `.xlsx`, `.epub`, or `.apk` +- web fonts (`font/woff`, `font/woff2`) -Text-based types are still compressed even under those prefixes, so `image/svg+xml` is compressed as usual. Set an accurate `content_type` when uploading media to a key-value store to benefit from this: +Two kinds of media type are compressed anyway: raw formats such as `image/bmp`, `image/tiff`, and `audio/wav`, and subtypes with a structured syntax suffix such as `image/svg+xml`. Set an accurate `content_type` when uploading media to a key-value store: @@ -40,7 +41,7 @@ Text-based types are still compressed even under those prefixes, so `image/svg+x -Without an explicit content type the record is sent as `application/octet-stream`, which the client cannot tell apart from uncompressed binary data and therefore still compresses. +Without an explicit content type, a `bytes` value is sent as `application/octet-stream`, which the client can't tell apart from uncompressed binary data and therefore still compresses. File-like values are streamed rather than buffered, so they're never compressed regardless of their content type. ## Configuration diff --git a/docs/02_concepts/code/13_skip_compression_async.py b/docs/02_concepts/code/13_skip_compression_async.py index ce7847f5..7d575645 100644 --- a/docs/02_concepts/code/13_skip_compression_async.py +++ b/docs/02_concepts/code/13_skip_compression_async.py @@ -14,3 +14,7 @@ async def main() -> None: # The explicit content type lets the client skip compressing the PNG. await kvs_client.set_record('screenshot', screenshot, content_type='image/png') + + +if __name__ == '__main__': + asyncio.run(main()) From be16625a29ec25408d7bacfd949423dce9e9baea Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 3 Aug 2026 16:39:33 +0200 Subject: [PATCH 6/6] fix: Compress audio/L24 and audio/midi request bodies --- src/apify_client/_consts.py | 2 ++ tests/unit/test_utils.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/apify_client/_consts.py b/src/apify_client/_consts.py index b7775e07..b41bc3a4 100644 --- a/src/apify_client/_consts.py +++ b/src/apify_client/_consts.py @@ -68,6 +68,8 @@ 'audio/aiff', 'audio/basic', 'audio/l16', + 'audio/l24', + 'audio/midi', 'audio/vnd.wave', 'audio/wav', 'audio/wave', diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0064f1b6..c6c7c226 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -322,6 +322,8 @@ def test_encode_key_value_store_record_value_non_encodable_with_explicit_content pytest.param('image/bmp', True, id='raw bitmap under a compressed prefix'), pytest.param('image/tiff', True, id='tiff under a compressed prefix'), pytest.param('audio/wav', True, id='raw audio under a compressed prefix'), + pytest.param('audio/L24', True, id='raw pcm audio in its registered casing'), + pytest.param('audio/midi', True, id='midi event data under a compressed prefix'), pytest.param('image/png', False, id='image prefix'), pytest.param('video/mp4', False, id='video prefix'), pytest.param('audio/mpeg', False, id='audio prefix'),