Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@typespec/http-client-python"
---

Fix a bug where a TypedDict literal value that coincides with a Python builtin type name (e.g. `type: "type"`) was corrupted into `Literal["builtins.type"]` in the generated `types.py`. The builtin-shadowing workaround now ignores identifiers inside string literals (literal values and quoted forward references are left untouched) and detects shadowing against the actually-emitted annotation, so genuine sibling-builtin shadowing is still qualified while spurious `import builtins` statements are no longer emitted.
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,38 @@
)


# Matches a single- or double-quoted string literal (escape-aware). Used to split an
# annotation so that bare-identifier rewriting/detection skips text inside string literals
# such as ``Literal["type"]`` values or quoted forward references like ``"SomeModel"``.
_STRING_LITERAL_RE = re.compile(r"""('(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*")""")


def _annotation_references_builtin(annotation: str, name: str) -> bool:
"""Whether ``name`` appears as a bare (non-dotted) identifier outside any string literal."""
pattern = re.compile(rf"(?<!\.)\b{name}\b")
# re.split with a single capturing group yields code/string/code/string/... segments.
for index, segment in enumerate(_STRING_LITERAL_RE.split(annotation)):
if index % 2 == 0 and pattern.search(segment):
return True
return False


def _qualify_shadowed_builtins(annotation: str, shadowed: frozenset[str]) -> str:
"""Replace bare builtin type references with builtins.X when shadowed by a field name."""
"""Replace bare builtin type references with builtins.X when shadowed by a field name.

Only bare identifiers in real type-expression positions are rewritten; identifiers
inside string literals (e.g. ``Literal["type"]`` values, quoted forward references)
and already-dotted names are left untouched.
"""
if not shadowed:
return annotation
for name in shadowed:
annotation = re.sub(rf"\b{name}\b", f"builtins.{name}", annotation)
return annotation
segments = _STRING_LITERAL_RE.split(annotation)
for index in range(0, len(segments), 2): # even indices are code, odd are string literals
segment = segments[index]
for name in shadowed:
segment = re.sub(rf"(?<!\.)\b{name}\b", f"builtins.{name}", segment)
segments[index] = segment
return "".join(segments)


class TypesSerializer(BaseSerializer):
Expand Down Expand Up @@ -225,22 +250,27 @@ def has_keyword_wire_names(model: ModelType) -> bool:
"""Whether any property wire_name is a Python keyword or requires functional TypedDict form."""
return any(keyword.iskeyword(p.wire_name) or not p.wire_name.isidentifier() for p in model.properties)

@staticmethod
def get_shadowed_builtins(model: ModelType) -> frozenset[str]:
def get_shadowed_builtins(self, model: ModelType) -> frozenset[str]:
"""Return the set of builtin type names shadowed by property wire_names in this model.

Only includes a builtin if it is both used as a wire_name AND referenced
in a type annotation within the same model (otherwise no shadowing occurs).
Only includes a builtin if it is both used as a wire_name AND referenced as a bare
type in an annotation within the same model (otherwise no shadowing occurs). Detection
uses the exact ``TYPES_FILE`` annotation emitted for each property so that types which
change under the types-file serialization (e.g. ``bytes``/``datetime`` -> ``str``,
``decimal`` -> ``float``) are evaluated as they actually appear in the output.
"""
wire_builtins = {p.wire_name for p in model.properties if p.wire_name in _BUILTIN_TYPE_NAMES}
if not wire_builtins:
return frozenset()
# Check which of these builtins actually appear in type annotations
# Check which of these builtins actually appear in emitted type annotations
used = set()
for prop in model.properties:
annotation = prop.type_annotation()
annotation = prop.type_annotation(
serialize_namespace=self.serialize_namespace,
serialize_namespace_type=NamespaceType.TYPES_FILE,
)
for name in wire_builtins:
if re.search(rf"\b{name}\b", annotation):
if _annotation_references_builtin(annotation, name):
used.add(name)
return frozenset(used)

Expand Down
146 changes: 145 additions & 1 deletion packages/http-client-python/tests/unit/test_typeddict.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pygen.codegen.models.model_type import TypedDictModelType
from pygen.codegen.models.property import Property
from pygen.codegen.models.list_type import ListType
from pygen.codegen.serializers.types_serializer import TypesSerializer
from pygen.codegen.serializers.types_serializer import TypesSerializer, _qualify_shadowed_builtins
from pygen.codegen.serializers.unions_serializer import UnionsSerializer


Expand Down Expand Up @@ -540,3 +540,147 @@ def test_enum_value_dpg_imports_enums_module():
enum_value = _make_enum_value(code_model)
modules = _local_import_modules(enum_value.imports())
assert any("_enums" in module for module in modules)


# ---------- builtin-shadowing qualification (Literal-value false positives) ----------


def _make_required_property(code_model, name, prop_type):
"""Create a required Property named ``name`` whose type is ``prop_type``."""
return Property(
yaml_data={"wireName": name, "clientName": name, "optional": False},
code_model=code_model,
type=prop_type,
)


def test_qualify_shadowed_builtins_ignores_literal_value():
"""A builtin name that only appears inside a ``Literal[...]`` value must not be rewritten."""
# ``type`` is a builtin, but here it is a literal string value, not a type reference.
assert _qualify_shadowed_builtins('Required[Literal["type"]]', frozenset({"type"})) == 'Required[Literal["type"]]'


def test_qualify_shadowed_builtins_ignores_quoted_forward_reference():
"""A builtin name inside a quoted forward reference must not be rewritten."""
assert _qualify_shadowed_builtins('list["type"]', frozenset({"type"})) == 'list["type"]'


def test_qualify_shadowed_builtins_rewrites_genuine_bare_reference():
"""A genuine bare builtin type reference is qualified as ``builtins.X``."""
assert _qualify_shadowed_builtins("bytes", frozenset({"bytes"})) == "builtins.bytes"
# Only the bare reference is rewritten; the quoted forward reference is left untouched.
assert _qualify_shadowed_builtins('Union["Model", bytes]', frozenset({"bytes"})) == 'Union["Model", builtins.bytes]'


def test_qualify_shadowed_builtins_skips_already_dotted_names():
"""An already-qualified ``builtins.X`` reference must not be double-qualified."""
assert _qualify_shadowed_builtins("builtins.bytes", frozenset({"bytes"})) == "builtins.bytes"


def test_qualify_shadowed_builtins_mixes_real_and_literal():
"""Within one annotation, a real reference is rewritten while a Literal value is preserved."""
assert (
_qualify_shadowed_builtins('dict[str, Literal["type"]]', frozenset({"type", "str"}))
== 'dict[builtins.str, Literal["type"]]'
)


def test_literal_value_matching_builtin_not_detected_as_shadowed():
"""A ``Literal["type"]`` value must not make ``type`` count as a shadowed builtin.

Regression: ``TypeParam`` has a ``type: Literal["type"]`` field and no sibling annotated
with the bare builtin ``type``, so no shadowing occurs.
"""
code_model = _make_code_model(models_mode="typeddict")
const = build_type({"type": "constant", "value": "type", "valueType": {"type": "string"}}, code_model)
text = build_type({"type": "string"}, code_model)
model = TypedDictModelType(
yaml_data={"name": "TypeParam", "type": "model", "snakeCaseName": "typeparam", "usage": 2},
code_model=code_model,
properties=[
_make_required_property(code_model, "type", const),
_make_required_property(code_model, "text", text),
],
)
env = _make_env()
ts = TypesSerializer(code_model=code_model, env=env, models=[model])
assert ts.get_shadowed_builtins(model) == frozenset()


def test_literal_value_false_positive_no_builtins_import_and_value_preserved():
"""The reported bug: ``Literal["type"]`` must stay intact and no ``import builtins`` is added."""
code_model = _make_code_model(models_mode="typeddict")
const = build_type({"type": "constant", "value": "type", "valueType": {"type": "string"}}, code_model)
text = build_type({"type": "string"}, code_model)
model = TypedDictModelType(
yaml_data={"name": "TypeParam", "type": "model", "snakeCaseName": "typeparam", "usage": 2},
code_model=code_model,
properties=[
_make_required_property(code_model, "type", const),
_make_required_property(code_model, "text", text),
],
)
code_model.model_types = [model]

env = _make_env()
output = TypesSerializer(code_model=code_model, env=env, models=[model]).serialize()
assert 'Literal["type"]' in output
assert "builtins.type" not in output
assert "import builtins" not in output


def test_genuine_sibling_builtin_is_detected_and_qualified():
"""A field whose wire name shadows a builtin used bare by a sibling annotation must qualify.

``Numbers`` has ``int: str`` (wire name ``int``) and ``count: int``. The bare ``int`` in
``count`` is emitted as-is in the types file, so pyright would resolve it to the earlier
field; it must be qualified as ``builtins.int``.
"""
code_model = _make_code_model(models_mode="typeddict")
str_type = build_type({"type": "string"}, code_model)
int_type = build_type({"type": "integer"}, code_model)
model = TypedDictModelType(
yaml_data={"name": "Numbers", "type": "model", "snakeCaseName": "numbers", "usage": 2},
code_model=code_model,
properties=[
_make_required_property(code_model, "int", str_type),
_make_required_property(code_model, "count", int_type),
],
)
code_model.model_types = [model]

env = _make_env()
ts = TypesSerializer(code_model=code_model, env=env, models=[model])
assert ts.get_shadowed_builtins(model) == frozenset({"int"})

output = ts.serialize()
assert "import builtins" in output
assert "count: Required[builtins.int]" in output


def test_type_changing_under_types_file_does_not_cause_spurious_builtins_import():
"""Detection must use the emitted annotation: a ``bytes`` field renders as ``str`` in types.py.

``Blob`` has ``bytes: int`` (wire name ``bytes``) and ``content: bytes``. In the types file a
bytes type is emitted as ``str``, so nothing genuinely references the bare builtin ``bytes``
and no ``import builtins`` should be added.
"""
code_model = _make_code_model(models_mode="typeddict")
int_type = build_type({"type": "integer"}, code_model)
bytes_type = build_type({"type": "bytes", "encode": "base64"}, code_model)
model = TypedDictModelType(
yaml_data={"name": "Blob", "type": "model", "snakeCaseName": "blob", "usage": 2},
code_model=code_model,
properties=[
_make_required_property(code_model, "bytes", int_type),
_make_required_property(code_model, "content", bytes_type),
],
)
code_model.model_types = [model]

env = _make_env()
ts = TypesSerializer(code_model=code_model, env=env, models=[model])
assert ts.get_shadowed_builtins(model) == frozenset()

output = ts.serialize()
assert "import builtins" not in output
Loading