Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions src/apify/storage_clients/_file_system/_key_value_store_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from typing_extensions import override

from crawlee._consts import METADATA_FILENAME
from crawlee._utils.file import atomic_write, infer_mime_type, json_dumps
from crawlee._utils.file import atomic_write, json_dumps
from crawlee.configuration import Configuration as CrawleeConfiguration
from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient
from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
Expand Down Expand Up @@ -125,21 +125,24 @@ async def _create_missing_metadata_for_input_file(key: str, record_path: Path) -
logger.warning('Input file disappeared from path: "%s"', record_path)
return

# Figure out the metadata from the file content
# Figure out the metadata from the file content. The parse and decode results are discarded: the calls only
# validate the content, so malformed input raises before any metadata is written.
size = len(content)
if record_path.suffix == '.json':
value = json.loads(content.decode('utf-8'))
json.loads(content.decode('utf-8'))
content_type = 'application/json; charset=utf-8'
elif record_path.suffix == '.txt':
value = content.decode('utf-8')
content.decode('utf-8')
content_type = 'text/plain; charset=utf-8'
elif record_path.suffix == '':
try:
value = json.loads(content.decode('utf-8'))
except json.JSONDecodeError:
value = content
json.loads(content.decode('utf-8'))
except (json.JSONDecodeError, UnicodeDecodeError):
content_type = 'application/octet-stream'
else:
content_type = 'application/json; charset=utf-8'
else:
value = content

content_type = infer_mime_type(value)
content_type = 'application/octet-stream'

record_metadata = KeyValueStoreRecordMetadata(key=key, content_type=content_type, size=size)
record_metadata_filepath = record_path.with_name(f'{record_path.name}.{METADATA_FILENAME}')
Expand Down
134 changes: 125 additions & 9 deletions tests/unit/storage_clients/test_file_system.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import asyncio
import json
from pathlib import Path
from typing import Any

import pytest

Expand Down Expand Up @@ -64,8 +65,23 @@ async def test_purge_preserves_input_file_and_metadata() -> None:
assert json.loads(input_content) == json.loads('{"test": "input"}')


@pytest.mark.parametrize('input_file_name', ['INPUT', 'INPUT.json'])
async def test_pre_existing_input_used_by_actor(input_file_name: str) -> None:
@pytest.mark.parametrize(
'input_file_name',
[pytest.param('INPUT', id='bare'), pytest.param('INPUT.json', id='json')],
)
@pytest.mark.parametrize(
'pre_existing_input',
Comment thread
vdusek marked this conversation as resolved.
Resolved
[
pytest.param({'foo': 'bar'}, id='object'),
pytest.param('', id='empty-string'),
pytest.param(0, id='zero-integer'),
pytest.param(1.5, id='float'),
pytest.param(False, id='false'),
pytest.param(None, id='null'),
Comment thread
vdusek marked this conversation as resolved.
Resolved
],
)
async def test_pre_existing_input_used_by_actor(input_file_name: str, pre_existing_input: Any) -> None:
"""Pre-existing JSON input preserves its value, type, bytes, and serialization metadata."""
configuration = Configuration()
service_locator.set_configuration(configuration)

Expand All @@ -74,15 +90,24 @@ async def test_pre_existing_input_used_by_actor(input_file_name: str) -> None:
path_to_input.mkdir(parents=True)
assert list(path_to_input.glob('*')) == []

pre_existing_input = {
'foo': 'bar',
}

# Create pre-existing INPUT.json file
(path_to_input / input_file_name).write_text(json.dumps(pre_existing_input))
input_file = path_to_input / input_file_name
input_bytes = json.dumps(pre_existing_input).encode('utf-8')
input_file.write_bytes(input_bytes)

async with Actor():
assert pre_existing_input == await Actor.get_input()
actual_input = await Actor.get_input()
assert actual_input == pre_existing_input
assert type(actual_input) is type(pre_existing_input)
store = await Actor.open_key_value_store()
stored_input = await store.get_value(configuration.input_key)
assert stored_input == pre_existing_input
assert type(stored_input) is type(pre_existing_input)

assert input_file.read_bytes() == input_bytes
metadata = json.loads((path_to_input / f'{input_file_name}.{METADATA_FILENAME}').read_text())
assert metadata['content_type'] == 'application/json; charset=utf-8'
assert metadata['key'] == configuration.input_key
assert metadata['size'] == len(input_bytes)

# Make sure that the input file doesn't get renamed in the process and metadata are added
assert set(path_to_input.glob('*')) == {
Expand All @@ -92,6 +117,35 @@ async def test_pre_existing_input_used_by_actor(input_file_name: str) -> None:
}


@pytest.mark.parametrize(
'input_bytes',
[pytest.param(b'hello', id='plain'), pytest.param('hello'.encode('utf-16'), id='utf16')],
)
async def test_pre_existing_non_json_input_used_by_actor(input_bytes: bytes) -> None:
"""Bare non-JSON input preserves its bytes and uses binary serialization metadata."""
configuration = Configuration()
service_locator.set_configuration(configuration)
path_to_input = Path(configuration.storage_dir) / 'key_value_stores' / 'default'
path_to_input.mkdir(parents=True)
input_file = path_to_input / 'INPUT'
input_file.write_bytes(input_bytes)

async with Actor():
actual_input = await Actor.get_input()
assert actual_input == input_bytes
assert type(actual_input) is bytes
store = await Actor.open_key_value_store()
stored_input = await store.get_value(configuration.input_key)
assert stored_input == input_bytes
assert type(stored_input) is bytes

assert input_file.read_bytes() == input_bytes
metadata = json.loads((path_to_input / f'INPUT.{METADATA_FILENAME}').read_text())
assert metadata['content_type'] == 'application/octet-stream'
assert metadata['key'] == configuration.input_key
assert metadata['size'] == len(input_bytes)


async def test_set_value_with_input_key_targets_existing_input_file() -> None:
"""`set_value` with the input key overwrites the existing `INPUT.json` instead of creating a duplicate."""
configuration = Configuration.get_global_configuration()
Expand Down Expand Up @@ -132,3 +186,65 @@ async def test_record_exists_and_delete_value_target_existing_input_file() -> No
await client.delete_value(key=configuration.input_key)
assert await client.record_exists(key=configuration.input_key) is False
assert not (kvs_path / 'INPUT.json').exists()


async def test_pre_existing_input_metadata_is_preserved() -> None:
"""Opening an input record respects its explicit content type and leaves its metadata unchanged."""
configuration = Configuration.get_global_configuration()
kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default'
kvs_path.mkdir(parents=True)
input_file = kvs_path / 'INPUT.json'
input_file.write_bytes(b'42')
metadata_file = kvs_path / f'INPUT.json.{METADATA_FILENAME}'
metadata_bytes = json.dumps(
{'key': configuration.input_key, 'content_type': 'text/plain; charset=utf-8', 'size': 2}
).encode('utf-8')
metadata_file.write_bytes(metadata_bytes)

client = await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration)
record = await client.get_value(key=configuration.input_key)

assert record is not None
assert record.value == '42'
assert input_file.read_bytes() == b'42'
assert metadata_file.read_bytes() == metadata_bytes


async def test_malformed_json_input_does_not_create_metadata() -> None:
"""Malformed JSON input raises without changing the input or creating record metadata."""
configuration = Configuration.get_global_configuration()
kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default'
kvs_path.mkdir(parents=True)
input_file = kvs_path / 'INPUT.json'
input_file.write_bytes(b'{not json')

with pytest.raises(json.JSONDecodeError):
await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration)

assert input_file.read_bytes() == b'{not json'
assert not (kvs_path / f'INPUT.json.{METADATA_FILENAME}').exists()


@pytest.mark.parametrize(
('input_file_name', 'expected_value', 'expected_content_type'),
[
pytest.param('INPUT.txt', '42', 'text/plain; charset=utf-8', id='txt'),
pytest.param('INPUT.bin', b'42', 'application/octet-stream', id='other-suffix'),
],
)
async def test_pre_existing_non_json_suffix_input_metadata(
input_file_name: str, expected_value: str | bytes, expected_content_type: str
) -> None:
"""Input with a `.txt` or other non-JSON suffix gets text or binary metadata from its suffix."""
configuration = Configuration.get_global_configuration()
kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default'
kvs_path.mkdir(parents=True)
(kvs_path / input_file_name).write_bytes(b'42')

client = await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration)
record = await client.get_value(key=configuration.input_key)

assert record is not None
assert record.value == expected_value
assert type(record.value) is type(expected_value)
assert record.content_type == expected_content_type
Loading