From 100a254e7c3ae6711ef6bd5139b6bd0dda665176 Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Mon, 31 Aug 2026 22:06:31 +0000 Subject: [PATCH 1/2] Fix PickleCoder.as_deterministic_coder() TypeError PickleCoder and _MemoizingPickleCoder construct FastPrimitivesCoder with a requires_deterministic kwarg the constructor has never accepted, raising TypeError on every call instead of returning a deterministic coder. Route both through the existing _update_compatible_deterministic_fast_primitives_coder helper, matching FastPrimitivesCoder's own as_deterministic_coder. Fixes #39942 --- CHANGES.md | 1 + sdks/python/apache_beam/coders/coders.py | 6 ++++-- sdks/python/apache_beam/coders/coders_test.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6357802c0eec..ece6a7e7d2be 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -94,6 +94,7 @@ * (Java) KafkaIO dynamic reads no longer require the obsolete `beam_fn_api` experiment ([#29998](https://github.com/apache/beam/issues/29998)). * (Prism) Self-checkpointing splittable DoFns now resume after their requested delay instead of immediately, so polling SDFs no longer busy-spin ([#39848](https://github.com/apache/beam/issues/39848)). * (Java) MongoDbIO read splitting now preserves non-ObjectId `_id` types (e.g. string ids) instead of failing to parse the generated range filters ([#39900](https://github.com/apache/beam/issues/39900)). +* (Python) Fixed `PickleCoder`/`_MemoizingPickleCoder.as_deterministic_coder()` raising `TypeError` instead of returning a working deterministic coder ([#28558](https://github.com/apache/beam/issues/28558)). ## Security Fixes diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 5f22bff5351b..cfdee3b1861c 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -899,7 +899,8 @@ def _nonhashable_dumps(x): return coder_impl.CallbackCoderImpl(_nonhashable_dumps, pickler.loads) def as_deterministic_coder(self, step_label, error_message=None): - return FastPrimitivesCoder(self, requires_deterministic=step_label) + return _update_compatible_deterministic_fast_primitives_coder( + self, step_label) def to_type_hint(self): return Any @@ -914,7 +915,8 @@ def _create_impl(self): lambda x: dumps(x, protocol), pickle.loads) def as_deterministic_coder(self, step_label, error_message=None): - return FastPrimitivesCoder(self, requires_deterministic=step_label) + return _update_compatible_deterministic_fast_primitives_coder( + self, step_label) def to_type_hint(self): return Any diff --git a/sdks/python/apache_beam/coders/coders_test.py b/sdks/python/apache_beam/coders/coders_test.py index ccd947457ad7..17fc46bd9f5e 100644 --- a/sdks/python/apache_beam/coders/coders_test.py +++ b/sdks/python/apache_beam/coders/coders_test.py @@ -50,6 +50,21 @@ def test_equality(self): self.assertNotEqual(coders.Base64PickleCoder(), coders.PickleCoder()) self.assertNotEqual(coders.Base64PickleCoder(), object()) + def test_as_deterministic_coder(self): + # PickleCoder.as_deterministic_coder used to construct FastPrimitivesCoder + # with a requires_deterministic kwarg that constructor never accepted, + # raising TypeError on every call (see coders.FastPrimitivesCoder). + v = ('a' * 10, 'b' * 90) + deterministic = coders.PickleCoder().as_deterministic_coder('label') + self.assertTrue(deterministic.is_deterministic()) + self.assertEqual(v, deterministic.decode(deterministic.encode(v))) + + memoizing_deterministic = coders._MemoizingPickleCoder( + ).as_deterministic_coder('label') + self.assertTrue(memoizing_deterministic.is_deterministic()) + self.assertEqual( + v, memoizing_deterministic.decode(memoizing_deterministic.encode(v))) + class CodersTest(unittest.TestCase): def test_str_utf8_coder(self): From fa9623c7217804ab13ba9baf6fdd3ee69df84ad9 Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Tue, 15 Sep 2026 17:52:46 +0000 Subject: [PATCH 2/2] Warn before falling back off PickleCoder to a deterministic coder Addresses review feedback on #39943: the deterministic fallback still fails at encode time for arbitrary classes, so log what changed and what it still cannot encode before the fallback is used. --- sdks/python/apache_beam/coders/coders.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index cfdee3b1861c..72d67cc9b551 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -38,6 +38,7 @@ import base64 import decimal +import logging import pickle from functools import lru_cache from typing import TYPE_CHECKING @@ -114,6 +115,8 @@ 'PaneInfoCoder' ] +_LOGGER = logging.getLogger(__name__) + T = TypeVar('T') CoderT = TypeVar('CoderT', bound='Coder') ProtoCoderT = TypeVar('ProtoCoderT', bound='ProtoCoder') @@ -899,6 +902,18 @@ def _nonhashable_dumps(x): return coder_impl.CallbackCoderImpl(_nonhashable_dumps, pickler.loads) def as_deterministic_coder(self, step_label, error_message=None): + _LOGGER.warning( + "PickleCoder was registered for a key type in '%s', but the runner " + "requires a deterministic key encoding and pickle is not " + "deterministic. Keys in this step will be encoded with the " + "deterministic fallback coder instead of pickle. That coder " + "supports primitives, containers, protobuf messages, frozen " + "dataclasses, NamedTuples, enums, and classes defining both " + "__getstate__ and __setstate__. Any other key type will fail at " + "encode time. If your key type is not one of these, register a " + "deterministic custom Coder for it or add a type hint so the " + "default coder is used.", + step_label) return _update_compatible_deterministic_fast_primitives_coder( self, step_label) @@ -915,6 +930,18 @@ def _create_impl(self): lambda x: dumps(x, protocol), pickle.loads) def as_deterministic_coder(self, step_label, error_message=None): + _LOGGER.warning( + "PickleCoder was registered for a key type in '%s', but the runner " + "requires a deterministic key encoding and pickle is not " + "deterministic. Keys in this step will be encoded with the " + "deterministic fallback coder instead of pickle. That coder " + "supports primitives, containers, protobuf messages, frozen " + "dataclasses, NamedTuples, enums, and classes defining both " + "__getstate__ and __setstate__. Any other key type will fail at " + "encode time. If your key type is not one of these, register a " + "deterministic custom Coder for it or add a type hint so the " + "default coder is used.", + step_label) return _update_compatible_deterministic_fast_primitives_coder( self, step_label)