From a7172e1932fda4386f47c40797dbc57ce52cc24d Mon Sep 17 00:00:00 2001 From: 1AbdullahButt <168034895+1AbdullahButt@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:58:57 +0500 Subject: [PATCH 1/2] Leave computed fields out of serialized pydantic models The pydantic transformer serialized models with model_dump_json(), which includes @computed_field values. Reading the value back with model_validate_json() then treats them as extra inputs, so a model with extra="forbid" could not be passed between tasks (extra_forbidden), and with the default extra="ignore" the computed values were dropped anyway. Pass exclude_computed_fields=True (pydantic >= 2.12) in both to_literal and the old generic format path; the receiving side computes the fields again when it builds the model. Closes flyteorg/flyte#8342 Co-Authored-By: Claude Opus 5.5 Signed-off-by: 1AbdullahButt <168034895+1AbdullahButt@users.noreply.github.com> --- .../pydantic_transformer/transformer.py | 14 ++++++- .../test_pydantic_basemodel_transformer.py | 40 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/flytekit/extras/pydantic_transformer/transformer.py b/flytekit/extras/pydantic_transformer/transformer.py index d609a9b775..a5000bfed6 100644 --- a/flytekit/extras/pydantic_transformer/transformer.py +++ b/flytekit/extras/pydantic_transformer/transformer.py @@ -1,3 +1,4 @@ +import inspect import json import os from typing import Type @@ -17,6 +18,15 @@ from flytekit.models.literals import Binary, Literal, Scalar from flytekit.models.types import LiteralType, TypeStructure +# Computed fields are derived again when the model is constructed, and pydantic rejects them as extra +# inputs for models with `extra="forbid"`, so leave them out of the serialized value. The option was +# added in pydantic 2.12. +_EXCLUDE_COMPUTED_FIELDS = ( + {"exclude_computed_fields": True} + if "exclude_computed_fields" in inspect.signature(BaseModel.model_dump_json).parameters + else {} +) + class PydanticTransformer(TypeTransformer[BaseModel]): def __init__(self): @@ -55,7 +65,7 @@ def to_generic_literal( """ Note: This is deprecated and will be removed in the future. """ - json_str = python_val.model_dump_json() + json_str = python_val.model_dump_json(**_EXCLUDE_COMPUTED_FIELDS) return Literal(scalar=Scalar(generic=_json_format.Parse(json_str, _struct.Struct()))) def to_literal( @@ -73,7 +83,7 @@ def to_literal( if str2bool(os.getenv(FLYTE_USE_OLD_DC_FORMAT)): return self.to_generic_literal(ctx, python_val, python_type, expected) - json_str = python_val.model_dump_json() + json_str = python_val.model_dump_json(**_EXCLUDE_COMPUTED_FIELDS) dict_obj = json.loads(json_str) msgpack_bytes = msgpack.dumps(dict_obj) return Literal(scalar=Scalar(binary=Binary(value=msgpack_bytes, tag=MESSAGEPACK))) diff --git a/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py b/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py index 7398cf2913..878d8d0da7 100644 --- a/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py +++ b/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py @@ -5,6 +5,7 @@ from typing import Dict, List, Optional, Union from unittest.mock import patch import mock +import pydantic import pytest from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct @@ -996,6 +997,45 @@ class BM(BaseModel): assert TypeEngine.to_literal_type(BM).annotation == TypeAnnotation({CACHE_KEY_METADATA: {SERIALIZATION_FORMAT: MESSAGEPACK}}) +@pytest.mark.skipif( + tuple(int(v) for v in pydantic.VERSION.split(".")[:2]) < (2, 12), + reason="model_dump_json(exclude_computed_fields=...) needs pydantic 2.12", +) +@pytest.mark.parametrize("use_old_dc_format", ["false", "true"]) +def test_basemodel_with_computed_field_and_extra_forbid(use_old_dc_format): + from pydantic import ConfigDict, computed_field + + class Inner(BaseModel): + model_config = ConfigDict(extra="forbid") + + n_days: int = 30 + + @computed_field + @property + def n_weeks(self) -> int: + return self.n_days // 7 + + class Outer(BaseModel): + model_config = ConfigDict(extra="forbid") + + inner: Inner = Field(default_factory=Inner) + name: str = "a" + + @computed_field + @property + def label(self) -> str: + return self.name.upper() + + ctx = FlyteContextManager.current_context() + lt = TypeEngine.to_literal_type(Outer) + with patch.dict(os.environ, {"FLYTE_USE_OLD_DC_FORMAT": use_old_dc_format}): + lv = TypeEngine.to_literal(ctx, Outer(inner=Inner(n_days=14)), Outer, lt) + pv = TypeEngine.to_python_value(ctx, lv, Outer) + assert pv == Outer(inner=Inner(n_days=14)) + assert pv.inner.n_weeks == 2 + assert pv.label == "A" + + @mock.patch("flytekit.remote.remote_fs.FlytePathResolver") def test_modify_literal_uris_call(mock_resolver): ctx = FlyteContextManager.current_context() From 3768cde51e7f03fbe908c253c20ce53adc396b69 Mon Sep 17 00:00:00 2001 From: 1AbdullahButt <168034895+1AbdullahButt@users.noreply.github.com> Date: Wed, 18 Nov 2026 04:41:19 +0500 Subject: [PATCH 2/2] Strip computed fields when reading pydantic models, not when writing Keep writing model_dump_json() as before, so stored outputs, cache keys and the UI keep the computed values. Before validating, the transformer now drops the computed-field keys of the expected model, walking nested models, lists, tuples, dicts, Optional and Annotated fields and RootModels. That also fixes outputs already stored by older versions, and works on every supported pydantic version. Co-Authored-By: Claude Opus 5.5 Signed-off-by: 1AbdullahButt <168034895+1AbdullahButt@users.noreply.github.com> --- .../pydantic_transformer/transformer.py | 77 ++++++++++++--- .../test_pydantic_basemodel_transformer.py | 93 +++++++++++++------ 2 files changed, 127 insertions(+), 43 deletions(-) diff --git a/flytekit/extras/pydantic_transformer/transformer.py b/flytekit/extras/pydantic_transformer/transformer.py index a5000bfed6..e7ebbb1856 100644 --- a/flytekit/extras/pydantic_transformer/transformer.py +++ b/flytekit/extras/pydantic_transformer/transformer.py @@ -1,12 +1,14 @@ -import inspect +import collections.abc import json import os -from typing import Type +import types as _pytypes +import typing +from typing import Annotated, Any, Type, Union import msgpack from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct -from pydantic import BaseModel +from pydantic import BaseModel, RootModel from flytekit import FlyteContext from flytekit.core.constants import CACHE_KEY_METADATA, FLYTE_USE_OLD_DC_FORMAT, MESSAGEPACK, SERIALIZATION_FORMAT @@ -18,14 +20,58 @@ from flytekit.models.literals import Binary, Literal, Scalar from flytekit.models.types import LiteralType, TypeStructure -# Computed fields are derived again when the model is constructed, and pydantic rejects them as extra -# inputs for models with `extra="forbid"`, so leave them out of the serialized value. The option was -# added in pydantic 2.12. -_EXCLUDE_COMPUTED_FIELDS = ( - {"exclude_computed_fields": True} - if "exclude_computed_fields" in inspect.signature(BaseModel.model_dump_json).parameters - else {} -) + +def _strip_computed_fields(tp: Any, data: Any) -> Any: + """ + Drop ``@computed_field`` values from serialized data before it is validated as ``tp``. + + ``model_dump_json()`` includes computed fields, so they are kept in stored outputs and shown in the UI. Pydantic + treats them as extra inputs when it reads the model back, which fails for models with ``extra="forbid"``, and + computes them again anyway. Stripping them on read also handles outputs that are already stored. + """ + origin = typing.get_origin(tp) + if origin is Annotated: + return _strip_computed_fields(typing.get_args(tp)[0], data) + if origin is Union or origin is _pytypes.UnionType: + candidates = [arg for arg in typing.get_args(tp) if arg is not type(None)] + # With several candidate types it isn't known which one the data is for, so leave it as it is. + return _strip_computed_fields(candidates[0], data) if len(candidates) == 1 else data + + if isinstance(tp, type) and issubclass(tp, BaseModel): + if issubclass(tp, RootModel): + return _strip_computed_fields(tp.model_fields["root"].annotation, data) + if not isinstance(data, dict): + return data + field_types = {} + for name, info in tp.model_fields.items(): + for key in (name, info.alias, info.validation_alias): + if isinstance(key, str): + field_types[key] = info.annotation + computed_keys = set() + for name, info in tp.model_computed_fields.items(): + computed_keys.update(key for key in (name, info.alias) if isinstance(key, str)) + return { + key: _strip_computed_fields(field_types[key], value) if key in field_types else value + for key, value in data.items() + if key in field_types or key not in computed_keys + } + + args = typing.get_args(tp) + if not args: + return data + if isinstance(data, list): + if origin is tuple: + if len(args) == 2 and args[1] is Ellipsis: + return [_strip_computed_fields(args[0], item) for item in data] + if len(args) != len(data): + return data + return [_strip_computed_fields(arg, item) for arg, item in zip(args, data)] + if isinstance(origin, type) and issubclass(origin, collections.abc.Iterable): + return [_strip_computed_fields(args[0], item) for item in data] + if isinstance(data, dict) and isinstance(origin, type) and issubclass(origin, collections.abc.Mapping): + if len(args) == 2: + return {key: _strip_computed_fields(args[1], value) for key, value in data.items()} + return data class PydanticTransformer(TypeTransformer[BaseModel]): @@ -65,7 +111,7 @@ def to_generic_literal( """ Note: This is deprecated and will be removed in the future. """ - json_str = python_val.model_dump_json(**_EXCLUDE_COMPUTED_FIELDS) + json_str = python_val.model_dump_json() return Literal(scalar=Scalar(generic=_json_format.Parse(json_str, _struct.Struct()))) def to_literal( @@ -83,7 +129,7 @@ def to_literal( if str2bool(os.getenv(FLYTE_USE_OLD_DC_FORMAT)): return self.to_generic_literal(ctx, python_val, python_type, expected) - json_str = python_val.model_dump_json(**_EXCLUDE_COMPUTED_FIELDS) + json_str = python_val.model_dump_json() dict_obj = json.loads(json_str) msgpack_bytes = msgpack.dumps(dict_obj) return Literal(scalar=Scalar(binary=Binary(value=msgpack_bytes, tag=MESSAGEPACK))) @@ -91,7 +137,7 @@ def to_literal( def from_binary_idl(self, binary_idl_object: Binary, expected_python_type: Type[BaseModel]) -> BaseModel: if binary_idl_object.tag == MESSAGEPACK: dict_obj = msgpack.loads(binary_idl_object.value, strict_map_key=False) - json_str = json.dumps(dict_obj) + json_str = json.dumps(_strip_computed_fields(expected_python_type, dict_obj)) python_val = expected_python_type.model_validate_json( json_data=json_str, strict=False, context={"deserialize": True} ) @@ -109,7 +155,8 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: if lv and lv.scalar and lv.scalar.binary is not None: return self.from_binary_idl(lv.scalar.binary, expected_python_type) # type: ignore - json_str = _json_format.MessageToJson(lv.scalar.generic) + dict_obj = json.loads(_json_format.MessageToJson(lv.scalar.generic)) + json_str = json.dumps(_strip_computed_fields(expected_python_type, dict_obj)) python_val = expected_python_type.model_validate_json(json_str, strict=False, context={"deserialize": True}) return python_val diff --git a/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py b/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py index 878d8d0da7..993254e894 100644 --- a/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py +++ b/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py @@ -5,11 +5,11 @@ from typing import Dict, List, Optional, Union from unittest.mock import patch import mock -import pydantic +import msgpack import pytest from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, computed_field from flytekit import task, workflow from flytekit.core.constants import CACHE_KEY_METADATA, MESSAGEPACK, SERIALIZATION_FORMAT @@ -997,45 +997,82 @@ class BM(BaseModel): assert TypeEngine.to_literal_type(BM).annotation == TypeAnnotation({CACHE_KEY_METADATA: {SERIALIZATION_FORMAT: MESSAGEPACK}}) -@pytest.mark.skipif( - tuple(int(v) for v in pydantic.VERSION.split(".")[:2]) < (2, 12), - reason="model_dump_json(exclude_computed_fields=...) needs pydantic 2.12", -) -@pytest.mark.parametrize("use_old_dc_format", ["false", "true"]) -def test_basemodel_with_computed_field_and_extra_forbid(use_old_dc_format): - from pydantic import ConfigDict, computed_field +class ComputedInner(BaseModel): + model_config = ConfigDict(extra="forbid") - class Inner(BaseModel): - model_config = ConfigDict(extra="forbid") + n_days: int = 30 - n_days: int = 30 + @computed_field + @property + def n_weeks(self) -> int: + return self.n_days // 7 - @computed_field - @property - def n_weeks(self) -> int: - return self.n_days // 7 - class Outer(BaseModel): - model_config = ConfigDict(extra="forbid") +class ComputedOuter(BaseModel): + model_config = ConfigDict(extra="forbid") - inner: Inner = Field(default_factory=Inner) - name: str = "a" + inner: ComputedInner = Field(default_factory=ComputedInner) + inners: List[ComputedInner] = Field(default_factory=list) + by_name: Dict[str, ComputedInner] = Field(default_factory=dict) + maybe: Optional[ComputedInner] = None + name: str = "a" - @computed_field - @property - def label(self) -> str: - return self.name.upper() + @computed_field + @property + def label(self) -> str: + return self.name.upper() + +@pytest.mark.parametrize("use_old_dc_format", ["false", "true"]) +def test_basemodel_with_computed_field_and_extra_forbid(use_old_dc_format): ctx = FlyteContextManager.current_context() - lt = TypeEngine.to_literal_type(Outer) + value = ComputedOuter( + inner=ComputedInner(n_days=14), + inners=[ComputedInner(n_days=7)], + by_name={"x": ComputedInner(n_days=21)}, + maybe=ComputedInner(n_days=28), + ) + lt = TypeEngine.to_literal_type(ComputedOuter) with patch.dict(os.environ, {"FLYTE_USE_OLD_DC_FORMAT": use_old_dc_format}): - lv = TypeEngine.to_literal(ctx, Outer(inner=Inner(n_days=14)), Outer, lt) - pv = TypeEngine.to_python_value(ctx, lv, Outer) - assert pv == Outer(inner=Inner(n_days=14)) + lv = TypeEngine.get_transformer(ComputedOuter).to_literal(ctx, value, ComputedOuter, lt) + + # Computed values stay in the stored output, so it is unchanged and they still show up in the UI. + if use_old_dc_format == "true": + stored = _json_format.MessageToDict(lv.scalar.generic) + else: + stored = msgpack.loads(lv.scalar.binary.value) + assert stored["label"] == "A" + assert stored["inner"]["n_weeks"] == 2 + assert stored["inners"][0]["n_weeks"] == 1 + + pv = TypeEngine.to_python_value(ctx, lv, ComputedOuter) + assert pv == value assert pv.inner.n_weeks == 2 + assert pv.by_name["x"].n_weeks == 3 + assert pv.maybe.n_weeks == 4 assert pv.label == "A" +def test_basemodel_with_computed_field_and_extra_forbid_between_tasks(): + @task + def make() -> ComputedInner: + return ComputedInner(n_days=14) + + @task + def make_list() -> List[ComputedInner]: + return [ComputedInner(n_days=7), ComputedInner(n_days=21)] + + @task + def weeks(p: ComputedInner, ps: List[ComputedInner]) -> List[int]: + return [p.n_weeks] + [x.n_weeks for x in ps] + + @workflow + def wf() -> List[int]: + return weeks(p=make(), ps=make_list()) + + assert wf() == [2, 1, 3] + + @mock.patch("flytekit.remote.remote_fs.FlytePathResolver") def test_modify_literal_uris_call(mock_resolver): ctx = FlyteContextManager.current_context()