Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 34 additions & 8 deletions lang/py/avro/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -1281,9 +1281,23 @@ def _read_default_value(self, field_schema: avro.schema.Schema, default_value: o
return None
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if field_schema.type == "boolean":
return bool(default_value)
if field_schema.type in ("int", "long"):
if isinstance(default_value, int):
if isinstance(default_value, bool):
return default_value
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if field_schema.type == "int":
if (
isinstance(default_value, int)
and not isinstance(default_value, bool)
and avro.schema.INT_MIN_VALUE <= default_value <= avro.schema.INT_MAX_VALUE
):
return default_value
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if field_schema.type == "long":
if (
isinstance(default_value, int)
and not isinstance(default_value, bool)
and avro.schema.LONG_MIN_VALUE <= default_value <= avro.schema.LONG_MAX_VALUE
):
return default_value
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if field_schema.type in ("float", "double"):
Expand All @@ -1296,28 +1310,40 @@ def _read_default_value(self, field_schema: avro.schema.Schema, default_value: o
if isinstance(default_value, str):
return default_value.encode()
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if field_schema.type in ("enum", "string"):
if isinstance(field_schema, avro.schema.EnumSchema):
if isinstance(default_value, str) and default_value in field_schema.symbols:
return default_value
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if field_schema.type == "string":
if isinstance(default_value, str):
return default_value
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if isinstance(field_schema, avro.schema.ArraySchema):
if isinstance(default_value, Iterable):
if isinstance(default_value, Iterable) and not isinstance(default_value, (str, bytes, Mapping)):
return [self._read_default_value(field_schema.items, json_val) for json_val in default_value]
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if isinstance(field_schema, avro.schema.MapSchema):
if isinstance(default_value, Mapping):
return {key: self._read_default_value(field_schema.values, json_val) for key, json_val in default_value.items()}
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if isinstance(field_schema, avro.schema.UnionSchema):
return self._read_default_value(field_schema.schemas[0], default_value)
for branch_schema in field_schema.schemas:
try:
return self._read_default_value(branch_schema, default_value)
except avro.errors.InvalidDefaultException:
continue
raise avro.errors.InvalidDefaultException(field_schema, default_value)
if isinstance(field_schema, avro.schema.RecordSchema):
if not isinstance(default_value, Mapping):
raise avro.errors.InvalidDefaultException(field_schema, default_value)
read_record = {}
for field in field_schema.fields:
json_val = default_value.get(field.name)
if json_val is None:
if field.name in default_value:
json_val = default_value[field.name]
elif field.has_default:
json_val = field.default
else:
raise avro.errors.InvalidDefaultException(field_schema, default_value)
field_val = self._read_default_value(field.type, json_val)
read_record[field.name] = field_val
return read_record
Expand Down
78 changes: 78 additions & 0 deletions lang/py/avro/test/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -1353,6 +1353,83 @@ def read(self, n: int = -1) -> bytes:
self.assertRaises(avro.errors.AvroCollectionSizeException, reader.read, decoder)


class TestDefaultValueResolution(unittest.TestCase):
"""Tests for record and union default value resolution in DatumReader."""

@staticmethod
def _read_evolved_default(field_schema: object, default_value: object) -> object:
writers_schema = avro.schema.parse(json.dumps({"type": "record", "name": "Test", "fields": []}))
readers_schema = avro.schema.parse(
json.dumps(
{
"type": "record",
"name": "Test",
"fields": [{"name": "f", "type": field_schema, "default": default_value}],
}
)
)
datum = cast(Dict[str, object], read_datum(io.BytesIO(b""), writers_schema, readers_schema))
return datum["f"]

def test_nested_record_explicit_null_default_not_overwritten(self) -> None:
inner_schema = {
"type": "record",
"name": "Inner",
"fields": [{"name": "x", "type": ["int", "null"], "default": 99}],
}
self.assertEqual(self._read_evolved_default(inner_schema, {"x": None}), {"x": None})
# Omitting the inner key still falls back to the inner field's default.
self.assertEqual(self._read_evolved_default(inner_schema, {}), {"x": 99})

def test_nested_record_omitted_field_with_null_default(self) -> None:
inner_schema = {
"type": "record",
"name": "InnerNull",
"fields": [{"name": "x", "type": ["int", "null"], "default": None}],
}
self.assertEqual(self._read_evolved_default(inner_schema, {}), {"x": None})

def test_nested_record_omitted_required_field_rejected(self) -> None:
inner_schema = {
"type": "record",
"name": "InnerRequired",
"fields": [{"name": "x", "type": ["null", "int"]}],
}
self.assertRaises(avro.errors.InvalidDefaultException, self._read_evolved_default, inner_schema, {})

def test_union_default_matches_non_first_branch(self) -> None:
self.assertEqual(self._read_evolved_default(["null", "string"], "hello"), "hello")
self.assertIsNone(self._read_evolved_default(["int", "null"], None))
self.assertEqual(self._read_evolved_default(["int", "long"], 1 << 40), 1 << 40)
self.assertIs(self._read_evolved_default(["int", "boolean"], True), True)
self.assertEqual(self._read_evolved_default(["boolean", "string"], "abc"), "abc")
self.assertEqual(self._read_evolved_default([{"type": "array", "items": "string"}, "string"], "abc"), "abc")
self.assertEqual(
self._read_evolved_default(
[
{"type": "enum", "name": "E1", "symbols": ["A"]},
{"type": "enum", "name": "E2", "symbols": ["B"]},
],
"B",
),
"B",
)
self.assertEqual(
self._read_evolved_default(
[
{"type": "record", "name": "R1", "fields": [{"name": "a", "type": "int"}]},
{"type": "record", "name": "R2", "fields": [{"name": "b", "type": "string"}]},
],
{"b": "val"},
),
{"b": "val"},
)

def test_union_default_rejects_unmatched_type(self) -> None:
self.assertRaises(avro.errors.InvalidDefaultException, self._read_evolved_default, ["int", "string"], None)
self.assertRaises(avro.errors.InvalidDefaultException, self._read_evolved_default, ["int", "string"], True)


def load_tests(loader: unittest.TestLoader, default_tests: None, pattern: None) -> unittest.TestSuite:
"""Generate test cases across many test schema."""
suite = unittest.TestSuite()
Expand All @@ -1370,6 +1447,7 @@ def load_tests(loader: unittest.TestLoader, default_tests: None, pattern: None)
suite.addTests(loader.loadTestsFromTestCase(TestBinaryDecoderAvailableBytes))
suite.addTests(loader.loadTestsFromTestCase(TestDatumReaderCollectionAvailableBytes))
suite.addTests(loader.loadTestsFromTestCase(TestDatumReaderCollectionSizeLimit))
suite.addTests(loader.loadTestsFromTestCase(TestDefaultValueResolution))
return suite


Expand Down
Loading