From 8c0f0a682f89d72c62ffbd699a033e5e57e8714f Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 3 Aug 2026 17:37:26 +0200 Subject: [PATCH 1/3] fix: Respect caller-supplied Content-Encoding for pre-compressed request bodies --- docs/02_concepts/13_http_compression.mdx | 27 +++++++- .../code/13_precompressed_async.py | 27 ++++++++ .../02_concepts/code/13_precompressed_sync.py | 22 +++++++ .../_resource_clients/key_value_store.py | 14 ++++ src/apify_client/http_clients/_base.py | 45 +++++++------ tests/unit/test_http_clients.py | 66 ++++++++++--------- tests/unit/test_key_value_store.py | 59 +++++++++++++++++ 7 files changed, 206 insertions(+), 54 deletions(-) create mode 100644 docs/02_concepts/code/13_precompressed_async.py create mode 100644 docs/02_concepts/code/13_precompressed_sync.py diff --git a/docs/02_concepts/13_http_compression.mdx b/docs/02_concepts/13_http_compression.mdx index b7f39733..09e5285d 100644 --- a/docs/02_concepts/13_http_compression.mdx +++ b/docs/02_concepts/13_http_compression.mdx @@ -10,12 +10,14 @@ import CodeBlock from '@theme/CodeBlock'; import SkipCompressionAsyncExample from '!!raw-loader!./code/13_skip_compression_async.py'; import SkipCompressionSyncExample from '!!raw-loader!./code/13_skip_compression_sync.py'; +import PrecompressedAsyncExample from '!!raw-loader!./code/13_precompressed_async.py'; +import PrecompressedSyncExample from '!!raw-loader!./code/13_precompressed_sync.py'; The Apify client compresses request bodies before sending them to the API. It reduces the amount of data transferred over the network, resulting in faster requests and lower bandwidth usage, especially for large payloads such as Actor inputs, dataset uploads, or key-value store records. ## How it works -The client compresses request bodies using the compressor configured via the `compression` parameter (default `'gzip'`). The server supports both gzip and brotli and decompresses the request body transparently. A body is compressed only when it is large enough to benefit and its content type isn't already compressed, as the next two sections describe. +The client compresses request bodies using the compressor configured via the `compression` parameter (default `'gzip'`). The server supports both gzip and brotli and decompresses the request body transparently. A body is compressed only when it's large enough to benefit, its content type isn't already compressed, and the request carries no `Content-Encoding` of its own. For details, see [Minimum body size](#minimum-body-size), [Already-compressed payloads](#already-compressed-payloads), and [Pre-compressed bodies](#pre-compressed-bodies). ## Minimum body size @@ -47,6 +49,27 @@ Two kinds of media type are compressed anyway: raw formats such as `image/bmp`, Without an explicit content type, a `bytes` value is sent as `application/octet-stream`, which the client can't tell apart from uncompressed binary data and therefore still compresses. File-like values are streamed rather than buffered, so they're never compressed regardless of their content type. +## Pre-compressed bodies + +A payload can reach the client already encoded, for example a gzipped file read from disk. Set the `Content-Encoding` header to name the encoding the payload carries. The client then sends the body as it is and forwards the header, so nothing gets compressed twice. `set_record` exposes the header as its `content_encoding` argument: + + + + + {PrecompressedAsyncExample} + + + + + {PrecompressedSyncExample} + + + + +The header is forwarded verbatim, so it also covers encodings the client ships no compressor for, such as `deflate`. The API accepts `gzip`, `br`, `deflate`, and `identity`. Passing `identity` turns compression off for a single request without changing how the client is configured. + +The client can't verify that the body matches the header, so set `Content-Encoding` only when the payload really is encoded that way. Key-value store records are stored exactly as you upload them, which makes the header part of the stored record rather than a transport detail. + ## Configuration To choose the compression algorithm, pass `compression` to the client constructor: @@ -88,7 +111,7 @@ client = ApifyClient(token='MY-APIFY-TOKEN', compression=BrotliHttpCompressor(qu client = ApifyClient(token='MY-APIFY-TOKEN', compression=GzipHttpCompressor(quality=9)) ``` -You can also implement a fully custom compressor by subclassing `HttpCompressor`. The client calls it only for bodies that reach the [minimum body size](#minimum-body-size) and aren't [already compressed](#already-compressed-payloads): +You can also implement a fully custom compressor by subclassing `HttpCompressor`. The client calls it only for bodies that reach the [minimum body size](#minimum-body-size) and aren't [already compressed](#already-compressed-payloads) or [pre-compressed by the caller](#pre-compressed-bodies): ```python from apify_client import ApifyClient diff --git a/docs/02_concepts/code/13_precompressed_async.py b/docs/02_concepts/code/13_precompressed_async.py new file mode 100644 index 00000000..95b12e95 --- /dev/null +++ b/docs/02_concepts/code/13_precompressed_async.py @@ -0,0 +1,27 @@ +import asyncio +import gzip +from pathlib import Path + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + kvs_client = apify_client.key_value_store('MY-KVS-ID') + + report = await asyncio.to_thread(Path('report.csv').read_bytes) + compressed_report = await asyncio.to_thread(gzip.compress, report) + + # The explicit content encoding stops the client from compressing the bytes again. + await kvs_client.set_record( + 'report', + compressed_report, + content_type='text/csv', + content_encoding='gzip', + ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/13_precompressed_sync.py b/docs/02_concepts/code/13_precompressed_sync.py new file mode 100644 index 00000000..a41d8b6c --- /dev/null +++ b/docs/02_concepts/code/13_precompressed_sync.py @@ -0,0 +1,22 @@ +import gzip +from pathlib import Path + +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + kvs_client = apify_client.key_value_store('MY-KVS-ID') + + report = Path('report.csv').read_bytes() + compressed_report = gzip.compress(report) + + # The explicit content encoding stops the client from compressing the bytes again. + kvs_client.set_record( + 'report', + compressed_report, + content_type='text/csv', + content_encoding='gzip', + ) diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 6e536be3..73c05605 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -360,6 +360,7 @@ def set_record( value: Any, *, content_type: str | None = None, + content_encoding: str | None = None, timeout: Timeout = 'long', ) -> None: """Set a value to the given record in the key-value store. @@ -370,11 +371,17 @@ def set_record( key: The key of the record to save the value to. value: The value to save into the record. content_type: The content type of the saved value. + content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it + to upload a pre-compressed value - the client then forwards the bytes as they are instead of + compressing them itself. The API accepts `gzip`, `br`, `deflate`, and `identity`, and stores the + record exactly as uploaded, so this also becomes the encoding the record is served with. timeout: Timeout for the API HTTP request. """ value, content_type = encode_key_value_store_record_value(value, content_type=content_type) headers = {'content-type': content_type} + if content_encoding is not None: + headers['content-encoding'] = content_encoding self._http_client.call( url=self._build_url(f'records/{key}'), @@ -776,6 +783,7 @@ async def set_record( value: Any, *, content_type: str | None = None, + content_encoding: str | None = None, timeout: Timeout = 'long', ) -> None: """Set a value to the given record in the key-value store. @@ -786,11 +794,17 @@ async def set_record( key: The key of the record to save the value to. value: The value to save into the record. content_type: The content type of the saved value. + content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it + to upload a pre-compressed value - the client then forwards the bytes as they are instead of + compressing them itself. The API accepts `gzip`, `br`, `deflate`, and `identity`, and stores the + record exactly as uploaded, so this also becomes the encoding the record is served with. timeout: Timeout for the API HTTP request. """ value, content_type = encode_key_value_store_record_value(value, content_type=content_type) headers = {'content-type': content_type} + if content_encoding is not None: + headers['content-encoding'] = content_encoding await self._http_client.call( url=self._build_url(f'records/{key}'), diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 6b9fc6d6..d25da24d 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -170,6 +170,11 @@ def _merge_headers(base: dict[str, str] | None, override: dict[str, str] | None) merged[key] = value return merged + @staticmethod + def _get_header(headers: dict[str, str], name: str) -> str | None: + """Look up a header value by name, treated case-insensitively. Returns `None` if the header is not set.""" + return next((value for key, value in headers.items() if key.lower() == name.lower()), None) + @staticmethod def _parse_params(params: dict[str, Any] | None) -> dict[str, Any] | None: """Convert request parameters to Apify API-compatible formats. @@ -228,9 +233,9 @@ def _compute_timeout(self, timeout: Timeout, *, attempt: int) -> int | float | N def _is_body_worth_compressing(data: str | bytes | bytearray | None) -> bool: """Whether this body clears the size threshold `_prepare_request_call` compresses at, cheaply. - Below the threshold nothing is ever compressed. At or above it the content type still decides, but - checking that here would buy nothing - a body that turns out to be already compressed only wastes the - thread hop this answer guards. + Below the threshold nothing is ever compressed. At or above it the content type and a caller-supplied + `Content-Encoding` still decide, but checking those here would buy nothing - a body that turns out to be + already encoded only wastes the thread hop this answer guards. The threshold is measured on encoded bytes, so a character count alone cannot decide a `str`. It is a lower bound, so a `str` long enough in characters is long enough in bytes too. Below that the encoded @@ -252,12 +257,15 @@ def _prepare_request_call( ) -> tuple[dict[str, str], dict[str, Any] | None, bytes | None]: """Prepare headers, params, and body for an HTTP request. - Merges the client's default headers (including authorization) with per-request headers, serializes JSON - and compresses the body unless it is smaller than `MIN_COMPRESSION_SIZE` or its content type says the - payload is already compressed. Header names are treated case-insensitively and per-request values win - over the client defaults. For JSON bodies, a `Content-Type` header is set unless the caller supplied one. - `Content-Encoding` always describes what was actually applied to the body, so a caller-supplied value is - dropped whenever nothing was compressed. + Merges the client's default headers (including authorization) with per-request headers and serializes a + JSON body. Header names are treated case-insensitively and per-request values win over the client + defaults. For JSON bodies, a `Content-Type` header is set unless the caller supplied one. + + The body is compressed unless a `Content-Encoding` header is already set, the body is smaller than + `MIN_COMPRESSION_SIZE`, or its content type says the payload is already compressed. A caller-supplied + `Content-Encoding` is forwarded verbatim, which is how a pre-encoded body is uploaded - including one in + an encoding the client ships no compressor for. `Content-Encoding: identity` therefore opts a single + request out of compression. """ if json is not None and data is not None: raise ValueError('Cannot pass both "json" and "data" parameters at the same time!') @@ -267,27 +275,24 @@ def _prepare_request_call( # Dump JSON data to a string so it can be sent as a request body. if json is not None: data = jsonlib.dumps(json, ensure_ascii=False, allow_nan=False, default=str).encode('utf-8') - if not any(key.lower() == 'content-type' for key in headers): + if self._get_header(headers, 'content-type') is None: headers['Content-Type'] = 'application/json' - compressed = False - if isinstance(data, (str, bytes, bytearray)): if isinstance(data, str): data = data.encode('utf-8') elif isinstance(data, bytearray): data = bytes(data) - content_type = next((value for key, value in headers.items() if key.lower() == 'content-type'), None) - if len(data) >= MIN_COMPRESSION_SIZE and is_compressible_content_type(content_type): + # A caller-supplied encoding says the body arrives already encoded, so compressing it here would + # both mislabel it and waste the work. + if ( + self._get_header(headers, 'content-encoding') is None + and len(data) >= MIN_COMPRESSION_SIZE + and is_compressible_content_type(self._get_header(headers, 'content-type')) + ): data = self._http_compressor.compress(data) headers = self._merge_headers(headers, {'Content-Encoding': self._http_compressor.content_encoding}) - compressed = True - - # Anything left uncompressed goes out as-is - a file-like body included - so a caller-supplied encoding - # would misdescribe it. - if data is not None and not compressed: - headers = {key: value for key, value in headers.items() if key.lower() != 'content-encoding'} return (headers, self._parse_params(params), data) diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index c52e7c2a..f0f88233 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -531,25 +531,10 @@ def test_prepare_request_call_skips_compression_for_already_compressed_content(c assert headers['User-Agent'] == client._headers['User-Agent'] -def test_prepare_request_call_drops_caller_content_encoding_when_compression_is_skipped() -> None: - """Skipping compression also strips a caller-supplied `Content-Encoding`, which would misdescribe the body.""" +def test_prepare_request_call_keeps_caller_content_encoding_for_a_streamed_body() -> None: + """A body the client streams rather than compresses, such as a file-like object, keeps its `Content-Encoding`.""" client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) - # Above the size threshold, so the content type is what skips compression here. - payload = b'\xff' * MIN_COMPRESSION_SIZE - - headers, _params, data = client._prepare_request_call( - headers={'content-type': 'image/jpeg', 'content-encoding': 'br'}, - data=payload, - ) - - assert data == payload - assert not any(key.lower() == 'content-encoding' for key in headers) - - -def test_prepare_request_call_drops_caller_content_encoding_for_a_streamed_body() -> None: - """A body that is streamed rather than compressed, such as a file-like object, also loses `Content-Encoding`.""" - client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) - stream = BytesIO(b'raw payload') + stream = BytesIO(gzip.compress(b'raw payload')) headers, _params, data = client._prepare_request_call( headers={'content-encoding': 'gzip'}, @@ -557,7 +542,7 @@ def test_prepare_request_call_drops_caller_content_encoding_for_a_streamed_body( ) assert data is stream - assert not any(key.lower() == 'content-encoding' for key in headers) + assert headers['content-encoding'] == 'gzip' @pytest.mark.parametrize( @@ -645,27 +630,44 @@ def test_prepare_request_call_json_keeps_caller_content_type() -> None: assert content_type_headers == {'content-type': 'application/json; charset=utf-8'} -def test_prepare_request_call_replaces_caller_content_encoding() -> None: - """A compressed body reports the compressor actually applied, replacing any caller-supplied Content-Encoding.""" +@pytest.mark.parametrize( + ('caller_headers', 'body'), + [ + pytest.param({'content-encoding': 'br'}, b'x' * MIN_COMPRESSION_SIZE, id='body the client would compress'), + pytest.param({'content-encoding': 'br'}, b'payload', id='body below the size threshold'), + pytest.param( + {'content-encoding': 'br', 'content-type': 'image/jpeg'}, + b'\xff' * MIN_COMPRESSION_SIZE, + id='already-compressed content type', + ), + pytest.param({'content-encoding': 'identity'}, b'x' * MIN_COMPRESSION_SIZE, id='identity opt-out'), + pytest.param( + {'content-encoding': 'deflate'}, + b'x' * MIN_COMPRESSION_SIZE, + id='encoding the client has no compressor for', + ), + ], +) +def test_prepare_request_call_keeps_caller_content_encoding(caller_headers: dict[str, str], body: bytes) -> None: + """A caller-supplied `Content-Encoding` marks the body as pre-encoded, so it goes out untouched and labeled.""" client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) - headers, _params, _data = client._prepare_request_call( - headers={'content-encoding': 'br'}, - data='x' * MIN_COMPRESSION_SIZE, - ) + headers, _params, data = client._prepare_request_call(headers=caller_headers, data=body) + assert data == body encoding_headers = {key: value for key, value in headers.items() if key.lower() == 'content-encoding'} - assert encoding_headers == {'Content-Encoding': 'gzip'} + assert encoding_headers == {'content-encoding': caller_headers['content-encoding']} -def test_prepare_request_call_drops_caller_content_encoding_when_skipping_compression() -> None: - """A caller-supplied Content-Encoding is dropped for an uncompressed body, so it cannot mislabel it.""" - client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) +def test_prepare_request_call_keeps_client_wide_content_encoding() -> None: + """A `Content-Encoding` configured on the client counts as caller-supplied on every request it sends.""" + client = _ConcreteHttpClient(headers={'Content-Encoding': 'identity'}, http_compressor=GzipHttpCompressor()) + body = b'x' * MIN_COMPRESSION_SIZE - headers, _params, data = client._prepare_request_call(headers={'content-encoding': 'br'}, data='payload') + headers, _params, data = client._prepare_request_call(data=body) - assert data == b'payload' - assert not any(key.lower() == 'content-encoding' for key in headers) + assert data == body + assert headers['Content-Encoding'] == 'identity' def test_build_url_with_params_none() -> None: diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index 5142f00c..7a88a7c9 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -2,6 +2,7 @@ import gzip import io +import zlib from typing import TYPE_CHECKING, Any import brotli @@ -42,6 +43,14 @@ def read(self) -> bytes: pytest.param(DuckTypedReader, _BYTES_VALUE, 'application/octet-stream', id='duck-typed reader'), ] +# Each case is (content encoding passed to `set_record`, the body the caller hands over already encoded that way). +_PRE_ENCODED_VALUE_CASES = [ + pytest.param('gzip', gzip.compress(_BYTES_VALUE), id='gzip'), + pytest.param('br', brotli.compress(_BYTES_VALUE), id='brotli'), + pytest.param('deflate', zlib.compress(_BYTES_VALUE), id='encoding the client has no compressor for'), + pytest.param('identity', _BYTES_VALUE, id='identity opt-out'), +] + @pytest.fixture( params=[ @@ -126,3 +135,53 @@ async def test_set_record_reads_file_like_value_async( assert captured_records[0].headers['content-encoding'] == content_encoding assert decode_body(captured_records[0]) == expected_body assert captured_records[0].headers['content-type'] == expected_content_type + + +@pytest.mark.parametrize(('content_encoding', 'value'), _PRE_ENCODED_VALUE_CASES) +def test_set_record_uploads_pre_encoded_value_sync( + *, + api_url: str, + captured_records: list[Request], + compression_case: tuple[HttpCompressionAlgorithm, str], + content_encoding: str, + value: bytes, +) -> None: + """An explicit `content_encoding` uploads the value as it is, whichever compressor the client uses.""" + algorithm, _client_encoding = compression_case + client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) + + client.key_value_store(_MOCKED_KVS_ID).set_record( + 'f', + value, + content_type='application/octet-stream', + content_encoding=content_encoding, + ) + + assert len(captured_records) == 1 + assert captured_records[0].headers['content-encoding'] == content_encoding + assert captured_records[0].get_data() == value + + +@pytest.mark.parametrize(('content_encoding', 'value'), _PRE_ENCODED_VALUE_CASES) +async def test_set_record_uploads_pre_encoded_value_async( + *, + api_url: str, + captured_records: list[Request], + compression_case: tuple[HttpCompressionAlgorithm, str], + content_encoding: str, + value: bytes, +) -> None: + """An explicit `content_encoding` uploads the value as it is, whichever compressor the client uses.""" + algorithm, _client_encoding = compression_case + client = ApifyClientAsync(token='test_token', api_url=api_url, compression=algorithm) + + await client.key_value_store(_MOCKED_KVS_ID).set_record( + 'f', + value, + content_type='application/octet-stream', + content_encoding=content_encoding, + ) + + assert len(captured_records) == 1 + assert captured_records[0].headers['content-encoding'] == content_encoding + assert captured_records[0].get_data() == value From f981ff290e287fd549cab7ddb2ad0df648ead38b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 3 Aug 2026 18:05:36 +0200 Subject: [PATCH 2/3] docs: Fix the file-like value claim in the HTTP compression guide --- docs/02_concepts/13_http_compression.mdx | 2 +- src/apify_client/http_clients/_base.py | 2 +- tests/unit/test_http_clients.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/02_concepts/13_http_compression.mdx b/docs/02_concepts/13_http_compression.mdx index 09e5285d..152aa741 100644 --- a/docs/02_concepts/13_http_compression.mdx +++ b/docs/02_concepts/13_http_compression.mdx @@ -47,7 +47,7 @@ Two kinds of media type are compressed anyway: raw formats such as `image/bmp`, -Without an explicit content type, a `bytes` value is sent as `application/octet-stream`, which the client can't tell apart from uncompressed binary data and therefore still compresses. File-like values are streamed rather than buffered, so they're never compressed regardless of their content type. +Without an explicit content type, a `bytes` value is sent as `application/octet-stream`, which the client can't tell apart from uncompressed binary data and therefore still compresses. File-like values are read into memory before they're sent, so they follow the same rules as any other body. ## Pre-compressed bodies diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index d25da24d..6f556bf0 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -153,7 +153,7 @@ def set_default_authorization(self, token: str) -> None: Args: token: The Apify API token to set as the `Bearer` authorization. """ - if not any(key.lower() == 'authorization' for key in self._headers): + if self._get_header(self._headers, 'authorization') is None: self._headers['Authorization'] = f'Bearer {token}' @staticmethod diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index f0f88233..34236887 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -531,8 +531,8 @@ def test_prepare_request_call_skips_compression_for_already_compressed_content(c assert headers['User-Agent'] == client._headers['User-Agent'] -def test_prepare_request_call_keeps_caller_content_encoding_for_a_streamed_body() -> None: - """A body the client streams rather than compresses, such as a file-like object, keeps its `Content-Encoding`.""" +def test_prepare_request_call_keeps_caller_content_encoding_for_a_file_like_body() -> None: + """A file-like body skips compression entirely, and its `Content-Encoding` reaches the transport untouched.""" client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor()) stream = BytesIO(gzip.compress(b'raw payload')) From 51ea1587697725408183b255f2debdc622108afa Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 4 Aug 2026 12:55:21 +0200 Subject: [PATCH 3/3] fix: Reject a declared compression for a value that cannot carry one --- docs/02_concepts/13_http_compression.mdx | 2 +- .../_resource_clients/key_value_store.py | 20 ++++++++-- src/apify_client/_utils/encoding.py | 20 +++++++++- tests/unit/test_key_value_store.py | 40 +++++++++++++++++++ tests/unit/test_utils.py | 39 +++++++++++++++++- 5 files changed, 113 insertions(+), 8 deletions(-) diff --git a/docs/02_concepts/13_http_compression.mdx b/docs/02_concepts/13_http_compression.mdx index 152aa741..a4d39ea6 100644 --- a/docs/02_concepts/13_http_compression.mdx +++ b/docs/02_concepts/13_http_compression.mdx @@ -68,7 +68,7 @@ A payload can reach the client already encoded, for example a gzipped file read The header is forwarded verbatim, so it also covers encodings the client ships no compressor for, such as `deflate`. The API accepts `gzip`, `br`, `deflate`, and `identity`. Passing `identity` turns compression off for a single request without changing how the client is configured. -The client can't verify that the body matches the header, so set `Content-Encoding` only when the payload really is encoded that way. Key-value store records are stored exactly as you upload them, which makes the header part of the stored record rather than a transport detail. +A value that can't be compressed at all - a string, an object serialized to JSON, or a file-like value opened in text mode - is rejected with a `TypeError` when `content_encoding` names a compression. Beyond that the client can't verify that the bytes match the header, so set `Content-Encoding` only when the payload really is encoded that way. Key-value store records are stored exactly as you upload them, which makes the header part of the stored record rather than a transport detail. ## Configuration diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 73c05605..b9c40325 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -374,10 +374,16 @@ def set_record( content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it to upload a pre-compressed value - the client then forwards the bytes as they are instead of compressing them itself. The API accepts `gzip`, `br`, `deflate`, and `identity`, and stores the - record exactly as uploaded, so this also becomes the encoding the record is served with. + record exactly as uploaded, so this also becomes the encoding the record is served with. Only a + bytes-like `value`, or a file-like one that reads into bytes, can carry a compression - anything + else raises `TypeError` instead of being stored under a header that misdescribes it. timeout: Timeout for the API HTTP request. """ - value, content_type = encode_key_value_store_record_value(value, content_type=content_type) + value, content_type = encode_key_value_store_record_value( + value, + content_type=content_type, + content_encoding=content_encoding, + ) headers = {'content-type': content_type} if content_encoding is not None: @@ -797,10 +803,16 @@ async def set_record( content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it to upload a pre-compressed value - the client then forwards the bytes as they are instead of compressing them itself. The API accepts `gzip`, `br`, `deflate`, and `identity`, and stores the - record exactly as uploaded, so this also becomes the encoding the record is served with. + record exactly as uploaded, so this also becomes the encoding the record is served with. Only a + bytes-like `value`, or a file-like one that reads into bytes, can carry a compression - anything + else raises `TypeError` instead of being stored under a header that misdescribes it. timeout: Timeout for the API HTTP request. """ - value, content_type = encode_key_value_store_record_value(value, content_type=content_type) + value, content_type = encode_key_value_store_record_value( + value, + content_type=content_type, + content_encoding=content_encoding, + ) headers = {'content-type': content_type} if content_encoding is not None: diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index 0c5722f0..39e5496f 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -13,7 +13,7 @@ def encode_key_value_store_record_value( - value: Any, *, content_type: str | None = None + value: Any, *, content_type: str | None = None, content_encoding: str | None = None ) -> tuple[bytes | bytearray | str, str]: """Encode a value for storage in a key-value store record. @@ -23,12 +23,17 @@ def encode_key_value_store_record_value( memory whole - the object is neither rewound nor closed, and async file-like objects are rejected. Any other value is JSON-serialized unless it is already bytes or a string. content_type: The content type; if None, it's inferred from the value type. + content_encoding: The encoding the caller declares the value already carries, if any. Anything other than + `identity` means the value is compressed, which only a bytes-like payload can be, so any other value + is rejected. The check belongs here because a file-like value has to be read before its payload type + is known, and reading it a second time in the caller is not possible. Returns: A tuple of (encoded_value, content_type). Raises: - TypeError: If the value cannot be encoded into a body the transport accepts. + TypeError: If the value cannot be encoded into a body the transport accepts, or if it cannot be carrying + the declared `content_encoding`. """ # Read file-like values into memory; the transport only accepts bytes-like bodies. Detect them by a # callable `read` (not `io.IOBase`) so duck-typed file-likes are read, not JSON-serialized. Impit exposes @@ -48,6 +53,17 @@ def encode_key_value_store_record_value( if not isinstance(value, (bytes, bytearray, str)): raise TypeError(f'Reading the file-like value returned {type(value).__name__}, expected bytes or str.') + # A declared compression describes bytes the caller compressed. A string, a JSON-serializable object, or a + # text-mode file cannot be carrying one, and would otherwise be stored under a header that misdescribes it - + # the client forwards the header untouched and never inspects the body. + declared_encoding = (content_encoding or '').strip().lower() + if declared_encoding not in ('', 'identity') and not isinstance(value, (bytes, bytearray)): + raise TypeError( + f'Cannot upload a {type(value).__name__} value with `Content-Encoding: {content_encoding}`. An encoding ' + 'other than `identity` declares the value is already compressed, so pass the compressed bytes, or a ' + 'file-like object that reads them.' + ) + if not content_type: if isinstance(value, (bytes, bytearray)): content_type = 'application/octet-stream' diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index 7a88a7c9..591540b5 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -51,6 +51,14 @@ def read(self) -> bytes: pytest.param('identity', _BYTES_VALUE, id='identity opt-out'), ] +# Values that cannot be carrying the `gzip` encoding the caller declares for them. Built by a factory for the +# same reason as `_FILE_LIKE_VALUE_CASES`, as the sync and async test each consume their own value. +_UNCOMPRESSIBLE_VALUE_CASES = [ + pytest.param(lambda: _TEXT_VALUE, id='string'), + pytest.param(lambda: {'key': 'value'}, id='json-serializable object'), + pytest.param(lambda: io.StringIO(_TEXT_VALUE), id='text-mode file-like'), +] + @pytest.fixture( params=[ @@ -185,3 +193,35 @@ async def test_set_record_uploads_pre_encoded_value_async( assert len(captured_records) == 1 assert captured_records[0].headers['content-encoding'] == content_encoding assert captured_records[0].get_data() == value + + +@pytest.mark.parametrize('make_value', _UNCOMPRESSIBLE_VALUE_CASES) +def test_set_record_rejects_declared_compression_of_non_bytes_value_sync( + *, + api_url: str, + captured_records: list[Request], + make_value: Callable[[], Any], +) -> None: + """A value that cannot be compressed is rejected before the request, not uploaded under a misleading header.""" + client = ApifyClient(token='test_token', api_url=api_url) + + with pytest.raises(TypeError, match='declares the value is already compressed'): + client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value(), content_encoding='gzip') + + assert captured_records == [] + + +@pytest.mark.parametrize('make_value', _UNCOMPRESSIBLE_VALUE_CASES) +async def test_set_record_rejects_declared_compression_of_non_bytes_value_async( + *, + api_url: str, + captured_records: list[Request], + make_value: Callable[[], Any], +) -> None: + """A value that cannot be compressed is rejected before the request, not uploaded under a misleading header.""" + client = ApifyClientAsync(token='test_token', api_url=api_url) + + with pytest.raises(TypeError, match='declares the value is already compressed'): + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value(), content_encoding='gzip') + + assert captured_records == [] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c6c7c226..61bbb3af 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,11 +1,12 @@ from __future__ import annotations +import gzip import io import json from base64 import b64decode from datetime import timedelta from http import HTTPStatus -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest.mock import Mock import impit @@ -28,6 +29,8 @@ from apify_client._typeddicts import WebhookRepresentationDict from apify_client.types import WebhooksList +_GZIPPED_DATA = gzip.compress(b'buffer data') + def test_to_safe_id() -> None: assert to_safe_id('abc') == 'abc' @@ -302,6 +305,40 @@ def read(self) -> None: encode_key_value_store_record_value(EmptyNonBlockingReader()) +@pytest.mark.parametrize( + ('value', 'expected_type_name'), + [ + pytest.param('already gzipped, honest', 'str', id='string'), + pytest.param({'a': 1}, 'dict', id='json-serializable object'), + pytest.param(io.StringIO('buffer data'), 'str', id='text-mode file-like'), + ], +) +def test_encode_key_value_store_record_value_declared_compression_of_non_bytes_raises( + value: Any, expected_type_name: str +) -> None: + """A value that cannot be compressed is rejected when the content encoding declares a compression.""" + with pytest.raises(TypeError, match=f'Cannot upload a {expected_type_name} value'): + encode_key_value_store_record_value(value, content_encoding='gzip') + + +@pytest.mark.parametrize( + ('value', 'content_encoding', 'expected_value'), + [ + pytest.param(_GZIPPED_DATA, 'gzip', _GZIPPED_DATA, id='bytes'), + pytest.param(bytearray(_GZIPPED_DATA), 'gzip', bytearray(_GZIPPED_DATA), id='bytearray'), + pytest.param(io.BytesIO(_GZIPPED_DATA), 'GZip', _GZIPPED_DATA, id='binary file-like, mixed-case encoding'), + pytest.param('buffer data', 'identity', 'buffer data', id='string under identity'), + pytest.param({'a': 1}, ' Identity ', b'{"a": 1}', id='json-serializable object under padded identity'), + ], +) +def test_encode_key_value_store_record_value_accepts_declared_encoding( + value: Any, content_encoding: str, expected_value: bytes | bytearray | str +) -> None: + """A bytes-like value passes the compression guard, and `identity` declares no compression at all.""" + encoded, _content_type = encode_key_value_store_record_value(value, content_encoding=content_encoding) + assert encoded == expected_value + + def test_encode_key_value_store_record_value_non_encodable_with_explicit_content_type_raises() -> None: """Test that a non-bytes-like value with a non-JSON content type is rejected before it reaches the transport.""" with pytest.raises(TypeError, match="Cannot encode a dict value as 'image/png'"):