diff --git a/.chronus/changes/fix-typeddict-literal-builtin-corruption-2026-7-15-8-47-0.md b/.chronus/changes/fix-typeddict-literal-builtin-corruption-2026-7-15-8-47-0.md new file mode 100644 index 00000000000..7cb17b79e9a --- /dev/null +++ b/.chronus/changes/fix-typeddict-literal-builtin-corruption-2026-7-15-8-47-0.md @@ -0,0 +1,7 @@ +--- +changeKind: fix +packages: + - "@typespec/http-client-python" +--- + +Fix a bug where a TypedDict literal value that coincides with a Python builtin type name (e.g. `type: "type"`) was corrupted into `Literal["builtins.type"]` in the generated `types.py`. The builtin-shadowing workaround now ignores identifiers inside string literals (literal values and quoted forward references are left untouched) and detects shadowing against the actually-emitted annotation, so genuine sibling-builtin shadowing is still qualified while spurious `import builtins` statements are no longer emitted. diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 30e1cb802e4..77cfcb112af 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -40,13 +40,38 @@ ) +# Matches a single- or double-quoted string literal (escape-aware). Used to split an +# annotation so that bare-identifier rewriting/detection skips text inside string literals +# such as ``Literal["type"]`` values or quoted forward references like ``"SomeModel"``. +_STRING_LITERAL_RE = re.compile(r"""('(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*")""") + + +def _annotation_references_builtin(annotation: str, name: str) -> bool: + """Whether ``name`` appears as a bare (non-dotted) identifier outside any string literal.""" + pattern = re.compile(rf"(? str: - """Replace bare builtin type references with builtins.X when shadowed by a field name.""" + """Replace bare builtin type references with builtins.X when shadowed by a field name. + + Only bare identifiers in real type-expression positions are rewritten; identifiers + inside string literals (e.g. ``Literal["type"]`` values, quoted forward references) + and already-dotted names are left untouched. + """ if not shadowed: return annotation - for name in shadowed: - annotation = re.sub(rf"\b{name}\b", f"builtins.{name}", annotation) - return annotation + segments = _STRING_LITERAL_RE.split(annotation) + for index in range(0, len(segments), 2): # even indices are code, odd are string literals + segment = segments[index] + for name in shadowed: + segment = re.sub(rf"(? bool: """Whether any property wire_name is a Python keyword or requires functional TypedDict form.""" return any(keyword.iskeyword(p.wire_name) or not p.wire_name.isidentifier() for p in model.properties) - @staticmethod - def get_shadowed_builtins(model: ModelType) -> frozenset[str]: + def get_shadowed_builtins(self, model: ModelType) -> frozenset[str]: """Return the set of builtin type names shadowed by property wire_names in this model. - Only includes a builtin if it is both used as a wire_name AND referenced - in a type annotation within the same model (otherwise no shadowing occurs). + Only includes a builtin if it is both used as a wire_name AND referenced as a bare + type in an annotation within the same model (otherwise no shadowing occurs). Detection + uses the exact ``TYPES_FILE`` annotation emitted for each property so that types which + change under the types-file serialization (e.g. ``bytes``/``datetime`` -> ``str``, + ``decimal`` -> ``float``) are evaluated as they actually appear in the output. """ wire_builtins = {p.wire_name for p in model.properties if p.wire_name in _BUILTIN_TYPE_NAMES} if not wire_builtins: return frozenset() - # Check which of these builtins actually appear in type annotations + # Check which of these builtins actually appear in emitted type annotations used = set() for prop in model.properties: - annotation = prop.type_annotation() + annotation = prop.type_annotation( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.TYPES_FILE, + ) for name in wire_builtins: - if re.search(rf"\b{name}\b", annotation): + if _annotation_references_builtin(annotation, name): used.add(name) return frozenset(used) diff --git a/packages/http-client-python/tests/unit/test_typeddict.py b/packages/http-client-python/tests/unit/test_typeddict.py index d4e1bd20845..3f26c2890fb 100644 --- a/packages/http-client-python/tests/unit/test_typeddict.py +++ b/packages/http-client-python/tests/unit/test_typeddict.py @@ -13,7 +13,7 @@ from pygen.codegen.models.model_type import TypedDictModelType from pygen.codegen.models.property import Property from pygen.codegen.models.list_type import ListType -from pygen.codegen.serializers.types_serializer import TypesSerializer +from pygen.codegen.serializers.types_serializer import TypesSerializer, _qualify_shadowed_builtins from pygen.codegen.serializers.unions_serializer import UnionsSerializer @@ -540,3 +540,147 @@ def test_enum_value_dpg_imports_enums_module(): enum_value = _make_enum_value(code_model) modules = _local_import_modules(enum_value.imports()) assert any("_enums" in module for module in modules) + + +# ---------- builtin-shadowing qualification (Literal-value false positives) ---------- + + +def _make_required_property(code_model, name, prop_type): + """Create a required Property named ``name`` whose type is ``prop_type``.""" + return Property( + yaml_data={"wireName": name, "clientName": name, "optional": False}, + code_model=code_model, + type=prop_type, + ) + + +def test_qualify_shadowed_builtins_ignores_literal_value(): + """A builtin name that only appears inside a ``Literal[...]`` value must not be rewritten.""" + # ``type`` is a builtin, but here it is a literal string value, not a type reference. + assert _qualify_shadowed_builtins('Required[Literal["type"]]', frozenset({"type"})) == 'Required[Literal["type"]]' + + +def test_qualify_shadowed_builtins_ignores_quoted_forward_reference(): + """A builtin name inside a quoted forward reference must not be rewritten.""" + assert _qualify_shadowed_builtins('list["type"]', frozenset({"type"})) == 'list["type"]' + + +def test_qualify_shadowed_builtins_rewrites_genuine_bare_reference(): + """A genuine bare builtin type reference is qualified as ``builtins.X``.""" + assert _qualify_shadowed_builtins("bytes", frozenset({"bytes"})) == "builtins.bytes" + # Only the bare reference is rewritten; the quoted forward reference is left untouched. + assert _qualify_shadowed_builtins('Union["Model", bytes]', frozenset({"bytes"})) == 'Union["Model", builtins.bytes]' + + +def test_qualify_shadowed_builtins_skips_already_dotted_names(): + """An already-qualified ``builtins.X`` reference must not be double-qualified.""" + assert _qualify_shadowed_builtins("builtins.bytes", frozenset({"bytes"})) == "builtins.bytes" + + +def test_qualify_shadowed_builtins_mixes_real_and_literal(): + """Within one annotation, a real reference is rewritten while a Literal value is preserved.""" + assert ( + _qualify_shadowed_builtins('dict[str, Literal["type"]]', frozenset({"type", "str"})) + == 'dict[builtins.str, Literal["type"]]' + ) + + +def test_literal_value_matching_builtin_not_detected_as_shadowed(): + """A ``Literal["type"]`` value must not make ``type`` count as a shadowed builtin. + + Regression: ``TypeParam`` has a ``type: Literal["type"]`` field and no sibling annotated + with the bare builtin ``type``, so no shadowing occurs. + """ + code_model = _make_code_model(models_mode="typeddict") + const = build_type({"type": "constant", "value": "type", "valueType": {"type": "string"}}, code_model) + text = build_type({"type": "string"}, code_model) + model = TypedDictModelType( + yaml_data={"name": "TypeParam", "type": "model", "snakeCaseName": "typeparam", "usage": 2}, + code_model=code_model, + properties=[ + _make_required_property(code_model, "type", const), + _make_required_property(code_model, "text", text), + ], + ) + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[model]) + assert ts.get_shadowed_builtins(model) == frozenset() + + +def test_literal_value_false_positive_no_builtins_import_and_value_preserved(): + """The reported bug: ``Literal["type"]`` must stay intact and no ``import builtins`` is added.""" + code_model = _make_code_model(models_mode="typeddict") + const = build_type({"type": "constant", "value": "type", "valueType": {"type": "string"}}, code_model) + text = build_type({"type": "string"}, code_model) + model = TypedDictModelType( + yaml_data={"name": "TypeParam", "type": "model", "snakeCaseName": "typeparam", "usage": 2}, + code_model=code_model, + properties=[ + _make_required_property(code_model, "type", const), + _make_required_property(code_model, "text", text), + ], + ) + code_model.model_types = [model] + + env = _make_env() + output = TypesSerializer(code_model=code_model, env=env, models=[model]).serialize() + assert 'Literal["type"]' in output + assert "builtins.type" not in output + assert "import builtins" not in output + + +def test_genuine_sibling_builtin_is_detected_and_qualified(): + """A field whose wire name shadows a builtin used bare by a sibling annotation must qualify. + + ``Numbers`` has ``int: str`` (wire name ``int``) and ``count: int``. The bare ``int`` in + ``count`` is emitted as-is in the types file, so pyright would resolve it to the earlier + field; it must be qualified as ``builtins.int``. + """ + code_model = _make_code_model(models_mode="typeddict") + str_type = build_type({"type": "string"}, code_model) + int_type = build_type({"type": "integer"}, code_model) + model = TypedDictModelType( + yaml_data={"name": "Numbers", "type": "model", "snakeCaseName": "numbers", "usage": 2}, + code_model=code_model, + properties=[ + _make_required_property(code_model, "int", str_type), + _make_required_property(code_model, "count", int_type), + ], + ) + code_model.model_types = [model] + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[model]) + assert ts.get_shadowed_builtins(model) == frozenset({"int"}) + + output = ts.serialize() + assert "import builtins" in output + assert "count: Required[builtins.int]" in output + + +def test_type_changing_under_types_file_does_not_cause_spurious_builtins_import(): + """Detection must use the emitted annotation: a ``bytes`` field renders as ``str`` in types.py. + + ``Blob`` has ``bytes: int`` (wire name ``bytes``) and ``content: bytes``. In the types file a + bytes type is emitted as ``str``, so nothing genuinely references the bare builtin ``bytes`` + and no ``import builtins`` should be added. + """ + code_model = _make_code_model(models_mode="typeddict") + int_type = build_type({"type": "integer"}, code_model) + bytes_type = build_type({"type": "bytes", "encode": "base64"}, code_model) + model = TypedDictModelType( + yaml_data={"name": "Blob", "type": "model", "snakeCaseName": "blob", "usage": 2}, + code_model=code_model, + properties=[ + _make_required_property(code_model, "bytes", int_type), + _make_required_property(code_model, "content", bytes_type), + ], + ) + code_model.model_types = [model] + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[model]) + assert ts.get_shadowed_builtins(model) == frozenset() + + output = ts.serialize() + assert "import builtins" not in output