diff --git a/flytekit/extras/pydantic_transformer/transformer.py b/flytekit/extras/pydantic_transformer/transformer.py index d609a9b775..e7ebbb1856 100644 --- a/flytekit/extras/pydantic_transformer/transformer.py +++ b/flytekit/extras/pydantic_transformer/transformer.py @@ -1,11 +1,14 @@ +import collections.abc import json import os -from typing import Type +import types as _pytypes +import typing +from typing import Annotated, Any, Type, Union import msgpack from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct -from pydantic import BaseModel +from pydantic import BaseModel, RootModel from flytekit import FlyteContext from flytekit.core.constants import CACHE_KEY_METADATA, FLYTE_USE_OLD_DC_FORMAT, MESSAGEPACK, SERIALIZATION_FORMAT @@ -18,6 +21,59 @@ from flytekit.models.types import LiteralType, TypeStructure +def _strip_computed_fields(tp: Any, data: Any) -> Any: + """ + Drop ``@computed_field`` values from serialized data before it is validated as ``tp``. + + ``model_dump_json()`` includes computed fields, so they are kept in stored outputs and shown in the UI. Pydantic + treats them as extra inputs when it reads the model back, which fails for models with ``extra="forbid"``, and + computes them again anyway. Stripping them on read also handles outputs that are already stored. + """ + origin = typing.get_origin(tp) + if origin is Annotated: + return _strip_computed_fields(typing.get_args(tp)[0], data) + if origin is Union or origin is _pytypes.UnionType: + candidates = [arg for arg in typing.get_args(tp) if arg is not type(None)] + # With several candidate types it isn't known which one the data is for, so leave it as it is. + return _strip_computed_fields(candidates[0], data) if len(candidates) == 1 else data + + if isinstance(tp, type) and issubclass(tp, BaseModel): + if issubclass(tp, RootModel): + return _strip_computed_fields(tp.model_fields["root"].annotation, data) + if not isinstance(data, dict): + return data + field_types = {} + for name, info in tp.model_fields.items(): + for key in (name, info.alias, info.validation_alias): + if isinstance(key, str): + field_types[key] = info.annotation + computed_keys = set() + for name, info in tp.model_computed_fields.items(): + computed_keys.update(key for key in (name, info.alias) if isinstance(key, str)) + return { + key: _strip_computed_fields(field_types[key], value) if key in field_types else value + for key, value in data.items() + if key in field_types or key not in computed_keys + } + + args = typing.get_args(tp) + if not args: + return data + if isinstance(data, list): + if origin is tuple: + if len(args) == 2 and args[1] is Ellipsis: + return [_strip_computed_fields(args[0], item) for item in data] + if len(args) != len(data): + return data + return [_strip_computed_fields(arg, item) for arg, item in zip(args, data)] + if isinstance(origin, type) and issubclass(origin, collections.abc.Iterable): + return [_strip_computed_fields(args[0], item) for item in data] + if isinstance(data, dict) and isinstance(origin, type) and issubclass(origin, collections.abc.Mapping): + if len(args) == 2: + return {key: _strip_computed_fields(args[1], value) for key, value in data.items()} + return data + + class PydanticTransformer(TypeTransformer[BaseModel]): def __init__(self): super().__init__("Pydantic Transformer", BaseModel, enable_type_assertions=False) @@ -81,7 +137,7 @@ def to_literal( def from_binary_idl(self, binary_idl_object: Binary, expected_python_type: Type[BaseModel]) -> BaseModel: if binary_idl_object.tag == MESSAGEPACK: dict_obj = msgpack.loads(binary_idl_object.value, strict_map_key=False) - json_str = json.dumps(dict_obj) + json_str = json.dumps(_strip_computed_fields(expected_python_type, dict_obj)) python_val = expected_python_type.model_validate_json( json_data=json_str, strict=False, context={"deserialize": True} ) @@ -99,7 +155,8 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: if lv and lv.scalar and lv.scalar.binary is not None: return self.from_binary_idl(lv.scalar.binary, expected_python_type) # type: ignore - json_str = _json_format.MessageToJson(lv.scalar.generic) + dict_obj = json.loads(_json_format.MessageToJson(lv.scalar.generic)) + json_str = json.dumps(_strip_computed_fields(expected_python_type, dict_obj)) python_val = expected_python_type.model_validate_json(json_str, strict=False, context={"deserialize": True}) return python_val diff --git a/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py b/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py index 7398cf2913..993254e894 100644 --- a/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py +++ b/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py @@ -5,10 +5,11 @@ from typing import Dict, List, Optional, Union from unittest.mock import patch import mock +import msgpack import pytest from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, computed_field from flytekit import task, workflow from flytekit.core.constants import CACHE_KEY_METADATA, MESSAGEPACK, SERIALIZATION_FORMAT @@ -996,6 +997,82 @@ class BM(BaseModel): assert TypeEngine.to_literal_type(BM).annotation == TypeAnnotation({CACHE_KEY_METADATA: {SERIALIZATION_FORMAT: MESSAGEPACK}}) +class ComputedInner(BaseModel): + model_config = ConfigDict(extra="forbid") + + n_days: int = 30 + + @computed_field + @property + def n_weeks(self) -> int: + return self.n_days // 7 + + +class ComputedOuter(BaseModel): + model_config = ConfigDict(extra="forbid") + + inner: ComputedInner = Field(default_factory=ComputedInner) + inners: List[ComputedInner] = Field(default_factory=list) + by_name: Dict[str, ComputedInner] = Field(default_factory=dict) + maybe: Optional[ComputedInner] = None + name: str = "a" + + @computed_field + @property + def label(self) -> str: + return self.name.upper() + + +@pytest.mark.parametrize("use_old_dc_format", ["false", "true"]) +def test_basemodel_with_computed_field_and_extra_forbid(use_old_dc_format): + ctx = FlyteContextManager.current_context() + value = ComputedOuter( + inner=ComputedInner(n_days=14), + inners=[ComputedInner(n_days=7)], + by_name={"x": ComputedInner(n_days=21)}, + maybe=ComputedInner(n_days=28), + ) + lt = TypeEngine.to_literal_type(ComputedOuter) + with patch.dict(os.environ, {"FLYTE_USE_OLD_DC_FORMAT": use_old_dc_format}): + lv = TypeEngine.get_transformer(ComputedOuter).to_literal(ctx, value, ComputedOuter, lt) + + # Computed values stay in the stored output, so it is unchanged and they still show up in the UI. + if use_old_dc_format == "true": + stored = _json_format.MessageToDict(lv.scalar.generic) + else: + stored = msgpack.loads(lv.scalar.binary.value) + assert stored["label"] == "A" + assert stored["inner"]["n_weeks"] == 2 + assert stored["inners"][0]["n_weeks"] == 1 + + pv = TypeEngine.to_python_value(ctx, lv, ComputedOuter) + assert pv == value + assert pv.inner.n_weeks == 2 + assert pv.by_name["x"].n_weeks == 3 + assert pv.maybe.n_weeks == 4 + assert pv.label == "A" + + +def test_basemodel_with_computed_field_and_extra_forbid_between_tasks(): + @task + def make() -> ComputedInner: + return ComputedInner(n_days=14) + + @task + def make_list() -> List[ComputedInner]: + return [ComputedInner(n_days=7), ComputedInner(n_days=21)] + + @task + def weeks(p: ComputedInner, ps: List[ComputedInner]) -> List[int]: + return [p.n_weeks] + [x.n_weeks for x in ps] + + @workflow + def wf() -> List[int]: + return weeks(p=make(), ps=make_list()) + + assert wf() == [2, 1, 3] + + @mock.patch("flytekit.remote.remote_fs.FlytePathResolver") def test_modify_literal_uris_call(mock_resolver): ctx = FlyteContextManager.current_context()