Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 61 additions & 4 deletions flytekit/extras/pydantic_transformer/transformer.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import collections.abc
import json
import os
from typing import Type
import types as _pytypes
import typing
from typing import Annotated, Any, Type, Union

import msgpack
from google.protobuf import json_format as _json_format
from google.protobuf import struct_pb2 as _struct
from pydantic import BaseModel
from pydantic import BaseModel, RootModel

from flytekit import FlyteContext
from flytekit.core.constants import CACHE_KEY_METADATA, FLYTE_USE_OLD_DC_FORMAT, MESSAGEPACK, SERIALIZATION_FORMAT
Expand All @@ -18,6 +21,59 @@
from flytekit.models.types import LiteralType, TypeStructure


def _strip_computed_fields(tp: Any, data: Any) -> Any:
"""
Drop ``@computed_field`` values from serialized data before it is validated as ``tp``.

``model_dump_json()`` includes computed fields, so they are kept in stored outputs and shown in the UI. Pydantic
treats them as extra inputs when it reads the model back, which fails for models with ``extra="forbid"``, and
computes them again anyway. Stripping them on read also handles outputs that are already stored.
"""
origin = typing.get_origin(tp)
if origin is Annotated:
return _strip_computed_fields(typing.get_args(tp)[0], data)
if origin is Union or origin is _pytypes.UnionType:
candidates = [arg for arg in typing.get_args(tp) if arg is not type(None)]
# With several candidate types it isn't known which one the data is for, so leave it as it is.
return _strip_computed_fields(candidates[0], data) if len(candidates) == 1 else data

if isinstance(tp, type) and issubclass(tp, BaseModel):
if issubclass(tp, RootModel):
return _strip_computed_fields(tp.model_fields["root"].annotation, data)
if not isinstance(data, dict):
return data
field_types = {}
for name, info in tp.model_fields.items():
for key in (name, info.alias, info.validation_alias):
if isinstance(key, str):
field_types[key] = info.annotation
computed_keys = set()
for name, info in tp.model_computed_fields.items():
computed_keys.update(key for key in (name, info.alias) if isinstance(key, str))
return {
key: _strip_computed_fields(field_types[key], value) if key in field_types else value
for key, value in data.items()
if key in field_types or key not in computed_keys
}

args = typing.get_args(tp)
if not args:
return data
if isinstance(data, list):
if origin is tuple:
if len(args) == 2 and args[1] is Ellipsis:
return [_strip_computed_fields(args[0], item) for item in data]
if len(args) != len(data):
return data
return [_strip_computed_fields(arg, item) for arg, item in zip(args, data)]
if isinstance(origin, type) and issubclass(origin, collections.abc.Iterable):
return [_strip_computed_fields(args[0], item) for item in data]
if isinstance(data, dict) and isinstance(origin, type) and issubclass(origin, collections.abc.Mapping):
if len(args) == 2:
return {key: _strip_computed_fields(args[1], value) for key, value in data.items()}
return data


class PydanticTransformer(TypeTransformer[BaseModel]):
def __init__(self):
super().__init__("Pydantic Transformer", BaseModel, enable_type_assertions=False)
Expand Down Expand Up @@ -81,7 +137,7 @@ def to_literal(
def from_binary_idl(self, binary_idl_object: Binary, expected_python_type: Type[BaseModel]) -> BaseModel:
if binary_idl_object.tag == MESSAGEPACK:
dict_obj = msgpack.loads(binary_idl_object.value, strict_map_key=False)
json_str = json.dumps(dict_obj)
json_str = json.dumps(_strip_computed_fields(expected_python_type, dict_obj))
python_val = expected_python_type.model_validate_json(
json_data=json_str, strict=False, context={"deserialize": True}
)
Expand All @@ -99,7 +155,8 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type:
if lv and lv.scalar and lv.scalar.binary is not None:
return self.from_binary_idl(lv.scalar.binary, expected_python_type) # type: ignore

json_str = _json_format.MessageToJson(lv.scalar.generic)
dict_obj = json.loads(_json_format.MessageToJson(lv.scalar.generic))
json_str = json.dumps(_strip_computed_fields(expected_python_type, dict_obj))
python_val = expected_python_type.model_validate_json(json_str, strict=False, context={"deserialize": True})
return python_val

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@
from typing import Dict, List, Optional, Union
from unittest.mock import patch
import mock
import msgpack
import pytest
from google.protobuf import json_format as _json_format
from google.protobuf import struct_pb2 as _struct
from pydantic import BaseModel, Field
from pydantic import BaseModel, ConfigDict, Field, computed_field

from flytekit import task, workflow
from flytekit.core.constants import CACHE_KEY_METADATA, MESSAGEPACK, SERIALIZATION_FORMAT
Expand Down Expand Up @@ -996,6 +997,82 @@ class BM(BaseModel):
assert TypeEngine.to_literal_type(BM).annotation == TypeAnnotation({CACHE_KEY_METADATA: {SERIALIZATION_FORMAT: MESSAGEPACK}})


class ComputedInner(BaseModel):
model_config = ConfigDict(extra="forbid")

n_days: int = 30

@computed_field
@property
def n_weeks(self) -> int:
return self.n_days // 7


class ComputedOuter(BaseModel):
model_config = ConfigDict(extra="forbid")

inner: ComputedInner = Field(default_factory=ComputedInner)
inners: List[ComputedInner] = Field(default_factory=list)
by_name: Dict[str, ComputedInner] = Field(default_factory=dict)
maybe: Optional[ComputedInner] = None
name: str = "a"

@computed_field
@property
def label(self) -> str:
return self.name.upper()


@pytest.mark.parametrize("use_old_dc_format", ["false", "true"])
def test_basemodel_with_computed_field_and_extra_forbid(use_old_dc_format):
ctx = FlyteContextManager.current_context()
value = ComputedOuter(
inner=ComputedInner(n_days=14),
inners=[ComputedInner(n_days=7)],
by_name={"x": ComputedInner(n_days=21)},
maybe=ComputedInner(n_days=28),
)
lt = TypeEngine.to_literal_type(ComputedOuter)
with patch.dict(os.environ, {"FLYTE_USE_OLD_DC_FORMAT": use_old_dc_format}):
lv = TypeEngine.get_transformer(ComputedOuter).to_literal(ctx, value, ComputedOuter, lt)

# Computed values stay in the stored output, so it is unchanged and they still show up in the UI.
if use_old_dc_format == "true":
stored = _json_format.MessageToDict(lv.scalar.generic)
else:
stored = msgpack.loads(lv.scalar.binary.value)
assert stored["label"] == "A"
assert stored["inner"]["n_weeks"] == 2
assert stored["inners"][0]["n_weeks"] == 1

pv = TypeEngine.to_python_value(ctx, lv, ComputedOuter)
assert pv == value
assert pv.inner.n_weeks == 2
assert pv.by_name["x"].n_weeks == 3
assert pv.maybe.n_weeks == 4
assert pv.label == "A"


def test_basemodel_with_computed_field_and_extra_forbid_between_tasks():
@task
def make() -> ComputedInner:
return ComputedInner(n_days=14)

@task
def make_list() -> List[ComputedInner]:
return [ComputedInner(n_days=7), ComputedInner(n_days=21)]

@task
def weeks(p: ComputedInner, ps: List[ComputedInner]) -> List[int]:
return [p.n_weeks] + [x.n_weeks for x in ps]

@workflow
def wf() -> List[int]:
return weeks(p=make(), ps=make_list())

assert wf() == [2, 1, 3]


@mock.patch("flytekit.remote.remote_fs.FlytePathResolver")
def test_modify_literal_uris_call(mock_resolver):
ctx = FlyteContextManager.current_context()
Expand Down