From c2ef59263b3be0c9bdc73f3ab0af65470da80595 Mon Sep 17 00:00:00 2001 From: xgbtxy <115146404+xgbtxy@users.noreply.github.com> Date: Sat, 14 Nov 2026 23:06:11 +0800 Subject: [PATCH] [python] Fix record and union default resolution In DatumReader._read_default_value, resolving a RecordSchema default used default_value.get(field.name) and replaced any None result with field.default. That silently overwrote explicit JSON null entries in nested record defaults (e.g. {"x": null}) whenever the inner field defined a non-null default, and fabricated None when a required inner field was omitted. In addition, UnionSchema defaults only tried schemas[0] instead of the first matching branch required by the Avro Specification ("Default values for union fields correspond to the first schema that matches in the union"), which also prevented explicit null or non-first-branch values in record and union defaults from resolving. Check key membership and field.has_default explicitly for RecordSchema defaults, and resolve UnionSchema defaults against the first matching branch with strict JSON type checks on primitive, enum, and array branches. Assisted-by: Google Antigravity --- lang/py/avro/io.py | 42 +++++++++++++++---- lang/py/avro/test/test_io.py | 78 ++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 8 deletions(-) diff --git a/lang/py/avro/io.py b/lang/py/avro/io.py index f5063a6686f..8b807ccb6ea 100644 --- a/lang/py/avro/io.py +++ b/lang/py/avro/io.py @@ -1281,9 +1281,23 @@ def _read_default_value(self, field_schema: avro.schema.Schema, default_value: o return None raise avro.errors.InvalidDefaultException(field_schema, default_value) if field_schema.type == "boolean": - return bool(default_value) - if field_schema.type in ("int", "long"): - if isinstance(default_value, int): + if isinstance(default_value, bool): + return default_value + raise avro.errors.InvalidDefaultException(field_schema, default_value) + if field_schema.type == "int": + if ( + isinstance(default_value, int) + and not isinstance(default_value, bool) + and avro.schema.INT_MIN_VALUE <= default_value <= avro.schema.INT_MAX_VALUE + ): + return default_value + raise avro.errors.InvalidDefaultException(field_schema, default_value) + if field_schema.type == "long": + if ( + isinstance(default_value, int) + and not isinstance(default_value, bool) + and avro.schema.LONG_MIN_VALUE <= default_value <= avro.schema.LONG_MAX_VALUE + ): return default_value raise avro.errors.InvalidDefaultException(field_schema, default_value) if field_schema.type in ("float", "double"): @@ -1296,12 +1310,16 @@ def _read_default_value(self, field_schema: avro.schema.Schema, default_value: o if isinstance(default_value, str): return default_value.encode() raise avro.errors.InvalidDefaultException(field_schema, default_value) - if field_schema.type in ("enum", "string"): + if isinstance(field_schema, avro.schema.EnumSchema): + if isinstance(default_value, str) and default_value in field_schema.symbols: + return default_value + raise avro.errors.InvalidDefaultException(field_schema, default_value) + if field_schema.type == "string": if isinstance(default_value, str): return default_value raise avro.errors.InvalidDefaultException(field_schema, default_value) if isinstance(field_schema, avro.schema.ArraySchema): - if isinstance(default_value, Iterable): + if isinstance(default_value, Iterable) and not isinstance(default_value, (str, bytes, Mapping)): return [self._read_default_value(field_schema.items, json_val) for json_val in default_value] raise avro.errors.InvalidDefaultException(field_schema, default_value) if isinstance(field_schema, avro.schema.MapSchema): @@ -1309,15 +1327,23 @@ def _read_default_value(self, field_schema: avro.schema.Schema, default_value: o return {key: self._read_default_value(field_schema.values, json_val) for key, json_val in default_value.items()} raise avro.errors.InvalidDefaultException(field_schema, default_value) if isinstance(field_schema, avro.schema.UnionSchema): - return self._read_default_value(field_schema.schemas[0], default_value) + for branch_schema in field_schema.schemas: + try: + return self._read_default_value(branch_schema, default_value) + except avro.errors.InvalidDefaultException: + continue + raise avro.errors.InvalidDefaultException(field_schema, default_value) if isinstance(field_schema, avro.schema.RecordSchema): if not isinstance(default_value, Mapping): raise avro.errors.InvalidDefaultException(field_schema, default_value) read_record = {} for field in field_schema.fields: - json_val = default_value.get(field.name) - if json_val is None: + if field.name in default_value: + json_val = default_value[field.name] + elif field.has_default: json_val = field.default + else: + raise avro.errors.InvalidDefaultException(field_schema, default_value) field_val = self._read_default_value(field.type, json_val) read_record[field.name] = field_val return read_record diff --git a/lang/py/avro/test/test_io.py b/lang/py/avro/test/test_io.py index 1a71e0598e5..a98d9954851 100644 --- a/lang/py/avro/test/test_io.py +++ b/lang/py/avro/test/test_io.py @@ -1353,6 +1353,83 @@ def read(self, n: int = -1) -> bytes: self.assertRaises(avro.errors.AvroCollectionSizeException, reader.read, decoder) +class TestDefaultValueResolution(unittest.TestCase): + """Tests for record and union default value resolution in DatumReader.""" + + @staticmethod + def _read_evolved_default(field_schema: object, default_value: object) -> object: + writers_schema = avro.schema.parse(json.dumps({"type": "record", "name": "Test", "fields": []})) + readers_schema = avro.schema.parse( + json.dumps( + { + "type": "record", + "name": "Test", + "fields": [{"name": "f", "type": field_schema, "default": default_value}], + } + ) + ) + datum = cast(Dict[str, object], read_datum(io.BytesIO(b""), writers_schema, readers_schema)) + return datum["f"] + + def test_nested_record_explicit_null_default_not_overwritten(self) -> None: + inner_schema = { + "type": "record", + "name": "Inner", + "fields": [{"name": "x", "type": ["int", "null"], "default": 99}], + } + self.assertEqual(self._read_evolved_default(inner_schema, {"x": None}), {"x": None}) + # Omitting the inner key still falls back to the inner field's default. + self.assertEqual(self._read_evolved_default(inner_schema, {}), {"x": 99}) + + def test_nested_record_omitted_field_with_null_default(self) -> None: + inner_schema = { + "type": "record", + "name": "InnerNull", + "fields": [{"name": "x", "type": ["int", "null"], "default": None}], + } + self.assertEqual(self._read_evolved_default(inner_schema, {}), {"x": None}) + + def test_nested_record_omitted_required_field_rejected(self) -> None: + inner_schema = { + "type": "record", + "name": "InnerRequired", + "fields": [{"name": "x", "type": ["null", "int"]}], + } + self.assertRaises(avro.errors.InvalidDefaultException, self._read_evolved_default, inner_schema, {}) + + def test_union_default_matches_non_first_branch(self) -> None: + self.assertEqual(self._read_evolved_default(["null", "string"], "hello"), "hello") + self.assertIsNone(self._read_evolved_default(["int", "null"], None)) + self.assertEqual(self._read_evolved_default(["int", "long"], 1 << 40), 1 << 40) + self.assertIs(self._read_evolved_default(["int", "boolean"], True), True) + self.assertEqual(self._read_evolved_default(["boolean", "string"], "abc"), "abc") + self.assertEqual(self._read_evolved_default([{"type": "array", "items": "string"}, "string"], "abc"), "abc") + self.assertEqual( + self._read_evolved_default( + [ + {"type": "enum", "name": "E1", "symbols": ["A"]}, + {"type": "enum", "name": "E2", "symbols": ["B"]}, + ], + "B", + ), + "B", + ) + self.assertEqual( + self._read_evolved_default( + [ + {"type": "record", "name": "R1", "fields": [{"name": "a", "type": "int"}]}, + {"type": "record", "name": "R2", "fields": [{"name": "b", "type": "string"}]}, + ], + {"b": "val"}, + ), + {"b": "val"}, + ) + + def test_union_default_rejects_unmatched_type(self) -> None: + self.assertRaises(avro.errors.InvalidDefaultException, self._read_evolved_default, ["int", "string"], None) + self.assertRaises(avro.errors.InvalidDefaultException, self._read_evolved_default, ["int", "string"], True) + + def load_tests(loader: unittest.TestLoader, default_tests: None, pattern: None) -> unittest.TestSuite: """Generate test cases across many test schema.""" suite = unittest.TestSuite() @@ -1370,6 +1447,7 @@ def load_tests(loader: unittest.TestLoader, default_tests: None, pattern: None) suite.addTests(loader.loadTestsFromTestCase(TestBinaryDecoderAvailableBytes)) suite.addTests(loader.loadTestsFromTestCase(TestDatumReaderCollectionAvailableBytes)) suite.addTests(loader.loadTestsFromTestCase(TestDatumReaderCollectionSizeLimit)) + suite.addTests(loader.loadTestsFromTestCase(TestDefaultValueResolution)) return suite