From 8292d74a897131c43b74df35f8ab386cc1fcaa0a Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sun, 27 Sep 2026 03:55:08 -0400 Subject: [PATCH 1/4] Add native Python symbols Map selected Smithy shapes to Python names and immutable type references. Apply service renames, detect normalized-name collisions, and resolve recursive types without recursive traversal. --- designs/codegen/index.md | 1 + designs/codegen/symbols.md | 137 +++++ .../smithy-python-feature-symbols.json | 4 + .../src/smithy_python/symbols.py | 215 ++++++++ .../smithy-python/tests/unit/test_symbols.py | 498 ++++++++++++++++++ 5 files changed, 855 insertions(+) create mode 100644 designs/codegen/symbols.md create mode 100644 packages/smithy-python/.changes/next-release/smithy-python-feature-symbols.json create mode 100644 packages/smithy-python/src/smithy_python/symbols.py create mode 100644 packages/smithy-python/tests/unit/test_symbols.py diff --git a/designs/codegen/index.md b/designs/codegen/index.md index 529286d49..eaf0fe73d 100644 --- a/designs/codegen/index.md +++ b/designs/codegen/index.md @@ -62,3 +62,4 @@ behavior of generated packages. * [Code Generator CLI](cli.md) * [Service and Data-Shape Selection](selection.md) +* [Native Python Symbols](symbols.md) diff --git a/designs/codegen/symbols.md b/designs/codegen/symbols.md new file mode 100644 index 000000000..52903e9cb --- /dev/null +++ b/designs/codegen/symbols.md @@ -0,0 +1,137 @@ +# Native Python Symbols + +`smithy_python.symbols` provides `SymbolProvider(model, selection, *, package)` and +frozen, hashable `TypeReference(name, module=None, arguments=(), nullable=False)`. +It uses only the standard library. Pass a loaded Model and its existing Selection; +it does not select shapes again, mutate either input, or generate files. + +## Contract + +* `declaration_name(id)` names selected structures, unions, enums and intEnums. + Only these supported declarations occupy `.models`. +* `member_name(member_id)` names structure/union fields or enum/intEnum constants. + List/map member names are not generated declarations and are rejected. +* `type_reference(id)` resolves a top-level data shape. To resolve a field's type, + pass its `MemberShape.target`, not its member ID. Requiredness, defaults and + field optionality are deliberately outside this API. + +IDs can be absolute strings or `ShapeId` values. Relative/malformed IDs raise +`InvalidShapeIdError`; missing shapes raise `ShapeNotFoundError`. Unselected +non-prelude shapes, control shapes, mixins, trait definitions, inappropriate +method requests and unsupported types raise `ModelError`. Prelude primitive +references and Unit work without selection. A manually narrowed Selection is +honored: references to omitted non-prelude targets fail rather than widening it. +The Model and Selection must describe the same loaded model. + +Package segments must be Python identifiers, not Python 3.12 hard keywords +(including their Unicode NFKC equivalents); invalid packages raise `ValueError`. +The supplied package spelling is preserved. Soft keywords are accepted. No +package is imported or checked for installation. + +```python +from smithy_python.model import load_model +from smithy_python.selection import select_shapes +from smithy_python.symbols import SymbolProvider, TypeReference + +model = load_model('''{ + "smithy": "2.0", + "shapes": { + "example#HTTPServer": { + "type": "structure", + "members": {"getURL": {"target": "smithy.api#String"}} + }, + "example#Servers": { + "type": "list", + "member": {"target": "example#HTTPServer"}, + "traits": {"smithy.api#sparse": {}} + } + } +}''') +symbols = SymbolProvider(model, select_shapes(model), package="example.client") +assert symbols.declaration_name("example#HTTPServer") == "HttpServer" +assert symbols.member_name("example#HTTPServer$getURL") == "get_url" +assert symbols.type_reference("example#Servers") == TypeReference( + "list", "builtins", + (TypeReference("HttpServer", "example.client.models", nullable=True),), +) +``` + +## Naming and collisions + +The selected service's rename is applied to declarations before normalization; +without a service the original shape name is used. IDs, member names in the +model, enum values and wire names are never changed. + +Split an uppercase run before its final capital when followed by lowercase; +then split lowercase-or-digit followed by uppercase. Underscores separate words: +empty words are discarded, including leading/trailing underscores. Digits stay +with the preceding word (except the digit-to-capital boundary). Lowercase words +are joined in PascalCase, snake_case or UPPER_SNAKE_CASE. Prefix `_` if the result +starts with a digit. Append `_` for an exact Python 3.12 hard keyword, using a +frozen explicit list, not the interpreter's keyword module. Soft keywords +`match`, `case`, `type`, `_` are not reserved. Special Python names such as +`__init__`, `_name_`, and `__private` lose their underscore wrappers, avoiding +magic methods, enum sunder names and name mangling. Empty or non-ASCII-identifier +rename words fail with the original ID; leading-digit renames are supported. + +| Input | Declaration | Field | Enum constant | +| --- | --- | --- | --- | +| HTTPServer | HttpServer | http_server | HTTP_SERVER | +| getURL | GetUrl | get_url | GET_URL | +| HTTP2Server | Http2Server | http2_server | HTTP2_SERVER | +| getURL2Value | GetUrl2Value | get_url2_value | GET_URL2_VALUE | +| __some__name__ | SomeName | some_name | SOME_NAME | +| __init__ | Init | init | INIT | +| class | Class | class_ | CLASS | +| None | None_ | none | NONE | +| match | Match | match | MATCH | + +For a leading-digit rename, `2HTTPServer` becomes `_2HttpServer`. + +Construction checks supported generated declarations in one module scope and +members in each separate declaration scope, after escaping. A collision raises +`ModelError` with both original IDs and the resulting Python name; the first +conflict follows selection and member order, independent of lookup order. +No numbering, builtin ban, speculative reservations or import-name collision +checks are performed. Primitive aliases and collections have no declarations. +Imported Document and generated Document retain different modules. + +## Type references and recursion + +`name` and `module` identify a type, `arguments` is an ordered tuple of nested +references, and `nullable=True` means this reference also permits None. The only +provider-produced reference without a module is `TypeReference("None")` for +Unit. There is no arbitrary metadata, import rendering or annotation-string API. + +| Smithy type | Symbolic Python type | +| --- | --- | +| boolean | builtins.bool | +| string | builtins.str | +| byte, short, integer, long, bigInteger | builtins.int | +| float, double | builtins.float | +| ordinary blob | builtins.bytes | +| bigDecimal | decimal.Decimal | +| timestamp | datetime.datetime | +| document | smithy_core.documents.Document | +| structure, union, enum, intEnum | `.models.` | +| list | builtins.list with one argument | +| map | builtins.dict with key and value arguments | +| smithy.api#Unit | None | + +Ordinary primitive aliases resolve to their underlying type. Sparse lists mark +only the element reference nullable; sparse maps mark only the value reference +nullable, including nested collections. These module names are symbolic strings: +codegen never imports runtime packages. + +Named references terminate traversal; self/mutual recursion and recursion through +collections do not create cyclic symbol objects. Collection expansion uses an +iterative postorder worklist with per-call results, not the Python call stack. +Collection-only cycles raise `ModelError` with the cycle IDs, excluding any +noncyclic prefix. Failed lookups cannot +poison subsequent resolutions. Streaming blobs and unions are rejected when +resolved (including through collections), not treated as ordinary types. They +do not reserve declarations at construction. Resolving a named structure does +not inspect its fields; consumers must resolve field targets separately. + +No service/operation/resource symbols, writers, CLI integration, schemas, +serializers, dependency tracking, plugins, or runtime dependencies are added. diff --git a/packages/smithy-python/.changes/next-release/smithy-python-feature-symbols.json b/packages/smithy-python/.changes/next-release/smithy-python-feature-symbols.json new file mode 100644 index 000000000..5b1d62984 --- /dev/null +++ b/packages/smithy-python/.changes/next-release/smithy-python-feature-symbols.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added a stdlib-only native Python symbol provider with deterministic declaration and member naming, structured type references, service renames, collision diagnostics, sparse collections, and recursive type support." +} diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py new file mode 100644 index 000000000..2466c8dde --- /dev/null +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -0,0 +1,215 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Python names and symbolic type references for an already selected model.""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass, replace + +from .model import ( + ListShape, + MapShape, + MemberShape, + Model, + ModelError, + Shape, + ShapeId, + ShapeType, +) +from .selection import Selection + +# Python 3.12 hard keywords, deliberately independent of the host interpreter. +# Soft keywords (match, case, type, _) remain ordinary identifiers here. +_KEYWORDS = frozenset( + "False None True and as assert async await break class continue def del " + "elif else except finally for from global if import in is lambda nonlocal " + "not or pass raise return try while with yield".split() +) +_ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) +_NAMED = _ENUMS | {ShapeType.STRUCTURE, ShapeType.UNION} + + +@dataclass(frozen=True, slots=True) +class TypeReference: + """A symbolic name, its defining module, and ordered generic arguments. + + ``module=None`` denotes the None literal. ``nullable`` permits None in + addition to this reference, and is used only for sparse collection entries. + No imports or annotation rendering are performed. + """ + + name: str + module: str | None = None + arguments: tuple[TypeReference, ...] = () + nullable: bool = False + + +_PRIMITIVES = { + ShapeType.BOOLEAN: TypeReference("bool", "builtins"), + ShapeType.STRING: TypeReference("str", "builtins"), + **dict.fromkeys( + ( + ShapeType.BYTE, + ShapeType.SHORT, + ShapeType.INTEGER, + ShapeType.LONG, + ShapeType.BIG_INTEGER, + ), + TypeReference("int", "builtins"), + ), + ShapeType.FLOAT: TypeReference("float", "builtins"), + ShapeType.DOUBLE: TypeReference("float", "builtins"), + ShapeType.BLOB: TypeReference("bytes", "builtins"), + ShapeType.BIG_DECIMAL: TypeReference("Decimal", "decimal"), + ShapeType.TIMESTAMP: TypeReference("datetime", "datetime"), + ShapeType.DOCUMENT: TypeReference("Document", "smithy_core.documents"), +} + + +def _name(value: str, *, pascal: bool = False, constant: bool = False) -> str: + value = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", value) + value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value) + words = [word.lower() for word in value.split("_") if word] + result = "".join(word.capitalize() for word in words) if pascal else "_".join(words) + if constant: + result = result.upper() + if result[:1].isdigit(): + result = "_" + result + if result in _KEYWORDS: + result += "_" + return result + + +class SymbolProvider: + """Naming for selected data shapes; IDs and wire names are never modified.""" + + def __init__(self, model: Model, selection: Selection, *, package: str) -> None: + if any( + not part.isidentifier() or unicodedata.normalize("NFKC", part) in _KEYWORDS + for part in package.split(".") + ): + raise ValueError(f"Invalid Python package name: {package!r}") + self._model = model + self._selection = selection + self._package = package + self._selected = frozenset(shape.id for shape in selection.shapes) + declarations: dict[str, ShapeId] = {} + for shape in selection.shapes: + if shape.type not in _NAMED or shape.has_trait("streaming"): + continue + self._check_collision( + declarations, self.declaration_name(shape.id), shape.id + ) + members: dict[str, ShapeId] = {} + for member in shape.members.values(): + self._check_collision(members, self.member_name(member.id), member.id) + + @staticmethod + def _check_collision( + scope: dict[str, ShapeId], name: str, shape_id: ShapeId + ) -> None: + if name in scope: + raise ModelError( + f"Python name collision {name!r}: {scope[name]}, {shape_id}" + ) + scope[name] = shape_id + + def _shape(self, shape_id: ShapeId | str) -> Shape: + shape = self._model.get_shape(shape_id) + if shape.id.root not in self._selected and not self._model.is_prelude(shape): + raise ModelError(f"Shape {shape.id} is not selected") + if ( + shape.is_mixin + or shape.has_trait("trait") + or shape.type + in (ShapeType.SERVICE, ShapeType.OPERATION, ShapeType.RESOURCE) + ): + raise ModelError(f"No data symbol for {shape.id} ({shape.type})") + if shape.type in (ShapeType.BLOB, ShapeType.UNION) and shape.has_trait( + "streaming" + ): + raise ModelError(f"Unsupported streaming {shape.type}: {shape.id}") + return shape + + def declaration_name(self, shape_id: ShapeId | str) -> str: + """Return a generated type's PascalCase name; aliases have no declaration.""" + shape = self._shape(shape_id) + if shape.type not in _NAMED or self._model.is_prelude(shape): + raise ModelError(f"No generated declaration for {shape.id}") + service = self._selection.service + name = service.rename.get(shape.id, shape.id.name) if service else shape.id.name + result = _name(name, pascal=True) + if not re.fullmatch(r"[A-Za-z0-9_]+", name) or not result.isidentifier(): + raise ModelError( + f"Invalid Python declaration name {result!r} for {shape.id}" + ) + return result + + def type_reference(self, shape_id: ShapeId | str) -> TypeReference: + """Resolve a data shape to an immutable symbolic Python type.""" + root = self._model.get_shape(shape_id).id + resolved: dict[ShapeId, TypeReference] = {} + active: dict[ShapeId, None] = {} + pending = [(root, False)] + while pending: + current, expanded = pending.pop() + if current in resolved: + continue + shape = self._shape(current) + if isinstance(shape, MemberShape): + raise ModelError( + f"Type references require a target, not member ID {current}" + ) + if isinstance(shape, (ListShape, MapShape)): + targets = ( + (shape.member.target,) + if isinstance(shape, ListShape) + else (shape.key.target, shape.value.target) + ) + if expanded: + arguments = tuple(resolved[target] for target in targets) + if shape.has_trait("sparse"): + arguments = ( + *arguments[:-1], + replace(arguments[-1], nullable=True), + ) + resolved[current] = TypeReference( + "list" if isinstance(shape, ListShape) else "dict", + "builtins", + arguments, + ) + del active[current] + else: + if current in active: + cycle = list(active) + cycle = [*cycle[cycle.index(current) :], current] + path = " -> ".join(str(item) for item in cycle) + raise ModelError(f"Collection-only cycle: {path}") + active[current] = None + pending.append((current, True)) + pending.extend((target, False) for target in reversed(targets)) + elif current == ShapeId("smithy.api", "Unit"): + resolved[current] = TypeReference("None") + elif shape.type in _NAMED: + resolved[current] = TypeReference( + self.declaration_name(current), f"{self._package}.models" + ) + elif shape.type in _PRIMITIVES: + resolved[current] = _PRIMITIVES[shape.type] + else: + raise ModelError( + f"No Python type reference for {current} ({shape.type})" + ) + return resolved[root] + + def member_name(self, shape_id: ShapeId | str) -> str: + """Return a field name or an enum constant, not a wire name.""" + member = self._shape(shape_id) + if not isinstance(member, MemberShape): + raise ModelError(f"Member naming requires a member ID: {member.id}") + container = self._shape(member.container) + if container.type not in _NAMED or self._model.is_prelude(container): + raise ModelError(f"No generated member name for {member.id}") + return _name(member.name, constant=container.type in _ENUMS) diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py new file mode 100644 index 000000000..926dcd9e3 --- /dev/null +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -0,0 +1,498 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest +from smithy_python.model import MemberShape, Model, load_model +from smithy_python.selection import select_shapes +from smithy_python.symbols import SymbolProvider + + +def model_with(shapes: dict[str, object]) -> Model: + return load_model(json.dumps({"smithy": "2.0", "shapes": shapes})) + + +@pytest.mark.parametrize( + ("source", "declaration", "field", "constant"), + [ + ("HTTPServer", "HttpServer", "http_server", "HTTP_SERVER"), + ("getURL", "GetUrl", "get_url", "GET_URL"), + ("HTTP2Server", "Http2Server", "http2_server", "HTTP2_SERVER"), + ("getURL2Value", "GetUrl2Value", "get_url2_value", "GET_URL2_VALUE"), + ("__some__name__", "SomeName", "some_name", "SOME_NAME"), + ("__init__", "Init", "init", "INIT"), + ("_value", "Value", "value", "VALUE"), + ("class", "Class", "class_", "CLASS"), + ("None", "None_", "none", "NONE"), + ("match", "Match", "match", "MATCH"), + ], +) +def test_names(source: str, declaration: str, field: str, constant: str) -> None: + model = model_with( + { + f"example#{source}": { + "type": "structure", + "members": {source: {"target": "smithy.api#String"}}, + }, + "example#Choice": { + "type": "enum", + "members": { + source: { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "wire"}, + } + }, + }, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="example.client") + assert symbols.declaration_name(f"example#{source}") == declaration + assert symbols.member_name(f"example#{source}${source}") == field + assert symbols.member_name(f"example#Choice${source}") == constant + assert model.get_shape(f"example#{source}").id.name == source + assert model.get_shape(f"example#Choice${source}").get_trait("enumValue") == "wire" + + +@pytest.mark.parametrize( + ("kind", "name", "module"), + [ + ("boolean", "bool", "builtins"), + ("string", "str", "builtins"), + *[ + (kind, "int", "builtins") + for kind in ("byte", "short", "integer", "long", "bigInteger") + ], + ("float", "float", "builtins"), + ("double", "float", "builtins"), + ("blob", "bytes", "builtins"), + ("bigDecimal", "Decimal", "decimal"), + ("timestamp", "datetime", "datetime"), + ("document", "Document", "smithy_core.documents"), + *[ + (kind, "HttpServer", "example.client.models") + for kind in ("structure", "union", "enum", "intEnum") + ], + ], +) +def test_type_references(kind: str, name: str, module: str) -> None: + from dataclasses import FrozenInstanceError + + from smithy_python.symbols import TypeReference + + model = model_with({"example#HTTPServer": {"type": kind}}) + symbols = SymbolProvider(model, select_shapes(model), package="example.client") + ref = symbols.type_reference("example#HTTPServer") + assert ref == TypeReference(name, module) + assert hash(ref) == hash(TypeReference(name, module)) + with pytest.raises(FrozenInstanceError): + setattr(ref, "name", "Changed") + if module != "example.client.models": + prelude_name = kind[0].upper() + kind[1:] + assert symbols.type_reference(f"smithy.api#{prelude_name}") == ref + assert symbols.type_reference("smithy.api#Unit") == TypeReference("None") + + +def test_service_rename_only_changes_declaration() -> None: + from smithy_python.selection import Selection + from smithy_python.symbols import TypeReference + + model = model_with( + { + "example#Service": { + "type": "service", + "operations": [{"target": "example#Op"}], + "rename": {"example#Original": "HTTPServer"}, + }, + "example#Op": { + "type": "operation", + "input": {"target": "example#Original"}, + }, + "example#Original": {"type": "structure"}, + } + ) + selection = select_shapes(model) + symbols = SymbolProvider(model, selection, package="pkg") + assert symbols.declaration_name("example#Original") == "HttpServer" + assert symbols.type_reference("example#Original") == TypeReference( + "HttpServer", "pkg.models" + ) + without_service = SymbolProvider( + model, Selection(None, selection.shapes, 0), package="pkg" + ) + assert without_service.declaration_name("example#Original") == "Original" + assert str(selection.shapes[0].id) == "example#Original" + + +@pytest.mark.parametrize("sparse", [False, True]) +def test_collections_and_recursive_named_types(sparse: bool) -> None: + from smithy_python.symbols import TypeReference + + traits: dict[str, object] = {"smithy.api#sparse": {}} if sparse else {} + model = model_with( + { + "example#Nodes": { + "type": "list", + "member": {"target": "example#Node"}, + "traits": traits, + }, + "example#Index": { + "type": "map", + "key": {"target": "smithy.api#String"}, + "value": {"target": "example#Nodes"}, + "traits": traits, + }, + "example#Node": { + "type": "structure", + "members": { + "self": {"target": "example#Node"}, + "children": {"target": "example#Index"}, + "other": {"target": "example#Other"}, + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + }, + "defaulted": { + "target": "smithy.api#String", + "traits": {"smithy.api#default": ""}, + }, + }, + }, + "example#Other": { + "type": "union", + "members": {"node": {"target": "example#Node"}}, + }, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + node = TypeReference("Node", "pkg.models") + nodes = TypeReference( + "list", "builtins", (TypeReference("Node", "pkg.models", nullable=sparse),) + ) + index = TypeReference( + "dict", + "builtins", + ( + TypeReference("str", "builtins"), + TypeReference("list", "builtins", nodes.arguments, nullable=sparse), + ), + ) + expected = { + "example#Node": node, + "example#Nodes": nodes, + "example#Index": index, + "example#Other": TypeReference("Other", "pkg.models"), + } + for order in (tuple(expected), tuple(reversed(expected))): + for shape_id in order: + assert symbols.type_reference(shape_id) == expected[shape_id] + for member in ("required", "defaulted"): + target = model.expect_shape(f"example#Node${member}", MemberShape).target + assert symbols.type_reference(target) == TypeReference("str", "builtins") + + +@pytest.mark.parametrize("mutual", [False, True]) +def test_collection_only_cycles(mutual: bool) -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Loop": { + "type": "list", + "member": {"target": "example#Map" if mutual else "example#Loop"}, + }, + "example#Map": { + "type": "map", + "key": {"target": "smithy.api#String"}, + "value": {"target": "example#Loop"}, + }, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + for shape_id in ("example#Loop", "example#Map", "example#Loop"): + with pytest.raises(ModelError, match=r"Collection-only cycle.*example#Loop"): + symbols.type_reference(shape_id) + + +@pytest.mark.parametrize("kind", ["structure", "union", "enum", "intEnum"]) +@pytest.mark.parametrize( + "names", [("getURL", "get_url"), ("class", "class_"), ("__init__", "init")] +) +def test_member_collisions(kind: str, names: tuple[str, str]) -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Container": { + "type": kind, + "members": {name: {"target": "smithy.api#Unit"} for name in names}, + } + } + ) + with pytest.raises(ModelError) as error: + SymbolProvider(model, select_shapes(model), package="pkg") + for name in names: + assert f"example#Container${name}" in str(error.value) + result = "CLASS" if kind in ("enum", "intEnum") else "class_" + if names[0] == "class": + assert repr(result) in str(error.value) + + +def test_declaration_collisions_after_rename() -> None: + from smithy_python.model import ModelError, ServiceShape + from smithy_python.selection import Selection + + model = model_with( + { + "example#Service": { + "type": "service", + "rename": {"example#Second": "HTTPServer"}, + }, + "example#Http_Server": {"type": "structure"}, + "example#Second": {"type": "enum"}, + } + ) + selection = Selection( + model.expect_shape("example#Service", ServiceShape), + (model.get_shape("example#Http_Server"), model.get_shape("example#Second")), + 0, + ) + with pytest.raises(ModelError) as error: + SymbolProvider(model, selection, package="pkg") + assert "'HttpServer'" in str(error.value) + assert "example#Http_Server" in str(error.value) + assert "example#Second" in str(error.value) + + +def test_only_emitted_declarations_collide() -> None: + from smithy_python.symbols import TypeReference + + model = model_with( + { + "example#HTTPServer": {"type": "string"}, + "example#Http_Server": {"type": "structure"}, + "example#Document": { + "type": "structure", + "members": { + "list": {"target": "smithy.api#Document"}, + "dict": {"target": "smithy.api#String"}, + }, + }, + "example#List": {"type": "enum"}, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + assert symbols.type_reference("example#Document") == TypeReference( + "Document", "pkg.models" + ) + assert symbols.type_reference("smithy.api#Document") == TypeReference( + "Document", "smithy_core.documents" + ) + assert symbols.member_name("example#Document$list") == "list" + assert symbols.member_name("example#Document$dict") == "dict" + + +@pytest.mark.parametrize( + "package", + [ + "", + ".pkg", + "pkg.", + "pkg..models", + "1pkg", + "pkg-name", + "pkg.class", + "None", + "pkg/child", + "pkg.\uff43\uff4c\uff41\uff53\uff53", + ], +) +def test_invalid_package(package: str) -> None: + model = model_with({}) + with pytest.raises(ValueError, match="Python package"): + SymbolProvider(model, select_shapes(model), package=package) + + +@pytest.mark.parametrize( + "name,expected", + [("2HTTPServer", "_2HttpServer"), ("__init__", "Init"), ("None", "None_")], +) +def test_rename_normalization(name: str, expected: str) -> None: + from smithy_python.model import ServiceShape + from smithy_python.selection import Selection + + model = model_with( + { + "example#Service": {"type": "service", "rename": {"example#Data": name}}, + "example#Data": {"type": "structure"}, + } + ) + selection = Selection( + model.expect_shape("example#Service", ServiceShape), + (model.get_shape("example#Data"),), + 0, + ) + assert ( + SymbolProvider(model, selection, package="pkg").declaration_name("example#Data") + == expected + ) + + +@pytest.mark.parametrize("kind", ["blob", "union"]) +def test_streaming_rejected_when_resolved(kind: str) -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Stream": {"type": kind, "traits": {"smithy.api#streaming": {}}}, + "example#Streams": {"type": "list", "member": {"target": "example#Stream"}}, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + for shape_id in ("example#Stream", "example#Streams"): + with pytest.raises(ModelError, match=r"Unsupported streaming.*example#Stream"): + symbols.type_reference(shape_id) + with pytest.raises(ModelError, match="Unsupported streaming"): + symbols.declaration_name("example#Stream") + + +def test_request_boundaries() -> None: + from smithy_python.model import InvalidShapeIdError, ModelError, ShapeNotFoundError + from smithy_python.selection import Selection + + model = model_with( + { + "example#Data": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + }, + "example#Excluded": {"type": "structure"}, + "example#Alias": {"type": "string"}, + "example#Items": {"type": "list", "member": {"target": "example#Excluded"}}, + "example#Service": {"type": "service"}, + "example#Operation": {"type": "operation"}, + "example#Resource": {"type": "resource"}, + "example#Trait": {"type": "structure", "traits": {"smithy.api#trait": {}}}, + "example#Mixin": {"type": "structure", "traits": {"smithy.api#mixin": {}}}, + } + ) + selection = Selection( + None, + tuple( + model.get_shape(f"example#{name}") for name in ("Data", "Alias", "Items") + ), + 1, + ) + symbols = SymbolProvider(model, selection, package="pkg") + for method in ( + symbols.type_reference, + symbols.declaration_name, + symbols.member_name, + ): + with pytest.raises(InvalidShapeIdError): + method("Data") + with pytest.raises(ShapeNotFoundError): + method("example#Missing") + with pytest.raises(ModelError, match="not selected"): + method("example#Excluded") + for name in ("Service", "Operation", "Resource", "Trait", "Mixin"): + with pytest.raises(ModelError): + symbols.type_reference(f"example#{name}") + for shape_id in ("example#Alias", "example#Items", "smithy.api#Unit"): + with pytest.raises(ModelError, match="No generated declaration"): + symbols.declaration_name(shape_id) + with pytest.raises(ModelError, match="member ID"): + symbols.type_reference("example#Data$value") + with pytest.raises(ModelError, match="member ID"): + symbols.member_name("example#Data") + with pytest.raises(ModelError, match="not selected"): + symbols.type_reference("example#Items") + + +@pytest.mark.parametrize("rename", ["___", "bad-name", "", "has space"]) +def test_invalid_rename_has_original_id(rename: str) -> None: + from smithy_python.model import ModelError, ServiceShape + from smithy_python.selection import Selection + + model = model_with( + { + "example#Service": {"type": "service", "rename": {"example#Data": rename}}, + "example#Data": {"type": "structure"}, + } + ) + selection = Selection( + model.expect_shape("example#Service", ServiceShape), + (model.get_shape("example#Data"),), + 0, + ) + with pytest.raises(ModelError, match="example#Data"): + SymbolProvider(model, selection, package="pkg") + + +def test_deep_collection_cycle_is_not_python_recursion() -> None: + from smithy_python.model import ModelError + + shapes: dict[str, object] = { + f"example#List{i}": { + "type": "list", + "member": {"target": f"example#List{(i + 1) % 1100}"}, + } + for i in range(1100) + } + model = model_with(shapes) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + with pytest.raises(ModelError, match="Collection-only cycle"): + symbols.type_reference("example#List0") + + +def test_package_is_keyword_only() -> None: + model = model_with({}) + with pytest.raises(TypeError): + SymbolProvider(model, select_shapes(model), "pkg") # type: ignore + + +@pytest.mark.parametrize("members", [False, True]) +def test_collision_diagnostics_follow_model_order(members: bool) -> None: + from smithy_python.model import ModelError + + first, second = ( + ("example#Data$get_url", "example#Data$getURL") + if members + else ("z#Get_Url", "a#GetUrl") + ) + shapes: dict[str, object] = ( + { + "example#Data": { + "type": "structure", + "members": { + "get_url": {"target": "smithy.api#String"}, + "getURL": {"target": "smithy.api#String"}, + }, + } + } + if members + else {first: {"type": "structure"}, second: {"type": "structure"}} + ) + model = model_with(shapes) + with pytest.raises(ModelError) as error: + SymbolProvider(model, select_shapes(model), package="pkg") + message = str(error.value) + assert message.index(first) < message.index(second) + + +def test_cycle_diagnostic_excludes_noncyclic_prefix() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Prefix": {"type": "list", "member": {"target": "example#Loop"}}, + "example#Loop": {"type": "list", "member": {"target": "example#Other"}}, + "example#Other": {"type": "list", "member": {"target": "example#Loop"}}, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + with pytest.raises(ModelError) as error: + symbols.type_reference("example#Prefix") + assert str(error.value) == ( + "Collection-only cycle: example#Loop -> example#Other -> example#Loop" + ) From 31db22c2f72b2d84655c1948bd1d11db71860335 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sun, 27 Sep 2026 12:20:19 -0400 Subject: [PATCH 2/4] Preserve acronyms and allow unknown enum values --- designs/codegen/symbols.md | 36 +++++--- .../src/smithy_python/symbols.py | 16 ++-- .../smithy-python/tests/unit/test_symbols.py | 90 +++++++++++++++---- 3 files changed, 108 insertions(+), 34 deletions(-) diff --git a/designs/codegen/symbols.md b/designs/codegen/symbols.md index 52903e9cb..ecb49cbd6 100644 --- a/designs/codegen/symbols.md +++ b/designs/codegen/symbols.md @@ -48,11 +48,11 @@ model = load_model('''{ } }''') symbols = SymbolProvider(model, select_shapes(model), package="example.client") -assert symbols.declaration_name("example#HTTPServer") == "HttpServer" +assert symbols.declaration_name("example#HTTPServer") == "HTTPServer" assert symbols.member_name("example#HTTPServer$getURL") == "get_url" assert symbols.type_reference("example#Servers") == TypeReference( "list", "builtins", - (TypeReference("HttpServer", "example.client.models", nullable=True),), + (TypeReference("HTTPServer", "example.client.models", nullable=True),), ) ``` @@ -62,11 +62,13 @@ The selected service's rename is applied to declarations before normalization; without a service the original shape name is used. IDs, member names in the model, enum values and wire names are never changed. -Split an uppercase run before its final capital when followed by lowercase; -then split lowercase-or-digit followed by uppercase. Underscores separate words: -empty words are discarded, including leading/trailing underscores. Digits stay -with the preceding word (except the digit-to-capital boundary). Lowercase words -are joined in PascalCase, snake_case or UPPER_SNAKE_CASE. Prefix `_` if the result +For declarations, split on underscores, uppercase the first character of each +nonempty part, and join without changing the remaining capitals. Existing +acronyms are preserved; lowercase words do not acquire invented acronyms. +For fields and enum constants, split an uppercase run before its final capital +when followed by lowercase; then split lowercase-or-digit followed by uppercase. +Join lowercase words with underscores for fields, or uppercase words for constants. +Empty words are discarded, including leading/trailing underscores. Prefix `_` if the result starts with a digit. Append `_` for an exact Python 3.12 hard keyword, using a frozen explicit list, not the interpreter's keyword module. Soft keywords `match`, `case`, `type`, `_` are not reserved. Special Python names such as @@ -76,17 +78,17 @@ rename words fail with the original ID; leading-digit renames are supported. | Input | Declaration | Field | Enum constant | | --- | --- | --- | --- | -| HTTPServer | HttpServer | http_server | HTTP_SERVER | -| getURL | GetUrl | get_url | GET_URL | -| HTTP2Server | Http2Server | http2_server | HTTP2_SERVER | -| getURL2Value | GetUrl2Value | get_url2_value | GET_URL2_VALUE | +| HTTPServer | HTTPServer | http_server | HTTP_SERVER | +| getURL | GetURL | get_url | GET_URL | +| HTTP2Server | HTTP2Server | http2_server | HTTP2_SERVER | +| getURL2Value | GetURL2Value | get_url2_value | GET_URL2_VALUE | | __some__name__ | SomeName | some_name | SOME_NAME | | __init__ | Init | init | INIT | | class | Class | class_ | CLASS | | None | None_ | none | NONE | | match | Match | match | MATCH | -For a leading-digit rename, `2HTTPServer` becomes `_2HttpServer`. +For a leading-digit rename, `2HTTPServer` becomes `_2HTTPServer`. Construction checks supported generated declarations in one module scope and members in each separate declaration scope, after escaping. A collision raises @@ -113,11 +115,19 @@ Unit. There is no arbitrary metadata, import rendering or annotation-string API. | bigDecimal | decimal.Decimal | | timestamp | datetime.datetime | | document | smithy_core.documents.Document | -| structure, union, enum, intEnum | `.models.` | +| structure, union | `.models.` | +| enum | builtins.str | +| intEnum | builtins.int | | list | builtins.list with one argument | | map | builtins.dict with key and value arguments | | smithy.api#Unit | None | +Enum declarations still have names and constants, but value references use +`str` or `int`, including inside collections, so annotations permit unknown +future values. For example, `declaration_name(Color)` returns `Color` while +`type_reference(Color)` returns `TypeReference("str", "builtins")`. This does +not implement runtime deserialization or validation. + Ordinary primitive aliases resolve to their underlying type. Sparse lists mark only the element reference nullable; sparse maps mark only the value reference nullable, including nested collections. These module names are symbolic strings: diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index 2466c8dde..96ee89240 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -49,6 +49,8 @@ class TypeReference: _PRIMITIVES = { ShapeType.BOOLEAN: TypeReference("bool", "builtins"), ShapeType.STRING: TypeReference("str", "builtins"), + ShapeType.ENUM: TypeReference("str", "builtins"), + ShapeType.INT_ENUM: TypeReference("int", "builtins"), **dict.fromkeys( ( ShapeType.BYTE, @@ -69,10 +71,12 @@ class TypeReference: def _name(value: str, *, pascal: bool = False, constant: bool = False) -> str: - value = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", value) - value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value) - words = [word.lower() for word in value.split("_") if word] - result = "".join(word.capitalize() for word in words) if pascal else "_".join(words) + if pascal: + result = "".join(word[:1].upper() + word[1:] for word in value.split("_")) + else: + value = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", value) + value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value) + result = "_".join(word.lower() for word in value.split("_") if word) if constant: result = result.upper() if result[:1].isdigit(): @@ -148,7 +152,7 @@ def declaration_name(self, shape_id: ShapeId | str) -> str: return result def type_reference(self, shape_id: ShapeId | str) -> TypeReference: - """Resolve a data shape to an immutable symbolic Python type.""" + """Resolve a value type; enums use str/int to permit unknown values.""" root = self._model.get_shape(shape_id).id resolved: dict[ShapeId, TypeReference] = {} active: dict[ShapeId, None] = {} @@ -192,7 +196,7 @@ def type_reference(self, shape_id: ShapeId | str) -> TypeReference: pending.extend((target, False) for target in reversed(targets)) elif current == ShapeId("smithy.api", "Unit"): resolved[current] = TypeReference("None") - elif shape.type in _NAMED: + elif shape.type in (ShapeType.STRUCTURE, ShapeType.UNION): resolved[current] = TypeReference( self.declaration_name(current), f"{self._package}.models" ) diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index 926dcd9e3..e9496aaad 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -16,10 +16,12 @@ def model_with(shapes: dict[str, object]) -> Model: @pytest.mark.parametrize( ("source", "declaration", "field", "constant"), [ - ("HTTPServer", "HttpServer", "http_server", "HTTP_SERVER"), - ("getURL", "GetUrl", "get_url", "GET_URL"), - ("HTTP2Server", "Http2Server", "http2_server", "HTTP2_SERVER"), - ("getURL2Value", "GetUrl2Value", "get_url2_value", "GET_URL2_VALUE"), + ("HTTPServer", "HTTPServer", "http_server", "HTTP_SERVER"), + ("getURL", "GetURL", "get_url", "GET_URL"), + ("HTTP_Server", "HTTPServer", "http_server", "HTTP_SERVER"), + ("http_server", "HttpServer", "http_server", "HTTP_SERVER"), + ("HTTP2Server", "HTTP2Server", "http2_server", "HTTP2_SERVER"), + ("getURL2Value", "GetURL2Value", "get_url2_value", "GET_URL2_VALUE"), ("__some__name__", "SomeName", "some_name", "SOME_NAME"), ("__init__", "Init", "init", "INIT"), ("_value", "Value", "value", "VALUE"), @@ -69,9 +71,11 @@ def test_names(source: str, declaration: str, field: str, constant: str) -> None ("bigDecimal", "Decimal", "decimal"), ("timestamp", "datetime", "datetime"), ("document", "Document", "smithy_core.documents"), + ("enum", "str", "builtins"), + ("intEnum", "int", "builtins"), *[ - (kind, "HttpServer", "example.client.models") - for kind in ("structure", "union", "enum", "intEnum") + (kind, "HTTPServer", "example.client.models") + for kind in ("structure", "union") ], ], ) @@ -87,12 +91,68 @@ def test_type_references(kind: str, name: str, module: str) -> None: assert hash(ref) == hash(TypeReference(name, module)) with pytest.raises(FrozenInstanceError): setattr(ref, "name", "Changed") - if module != "example.client.models": + if module != "example.client.models" and kind not in ("enum", "intEnum"): prelude_name = kind[0].upper() + kind[1:] assert symbols.type_reference(f"smithy.api#{prelude_name}") == ref assert symbols.type_reference("smithy.api#Unit") == TypeReference("None") +@pytest.mark.parametrize("kind,primitive", [("enum", "str"), ("intEnum", "int")]) +@pytest.mark.parametrize("sparse", [False, True]) +def test_enum_value_references(kind: str, primitive: str, sparse: bool) -> None: + from smithy_python.symbols import TypeReference + + traits: dict[str, object] = {"smithy.api#sparse": {}} if sparse else {} + model = model_with( + { + "example#HTTPStatus": { + "type": kind, + "members": { + "OK": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ok" if kind == "enum" else 200 + }, + } + }, + }, + "example#Statuses": { + "type": "list", + "member": {"target": "example#HTTPStatus"}, + "traits": traits, + }, + "example#Index": { + "type": "map", + "key": {"target": "smithy.api#String"}, + "value": {"target": "example#Statuses"}, + "traits": traits, + }, + "example#Response": { + "type": "structure", + "members": {"status": {"target": "example#HTTPStatus"}}, + }, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + assert symbols.declaration_name("example#HTTPStatus") == "HTTPStatus" + assert symbols.member_name("example#HTTPStatus$OK") == "OK" + target = model.expect_shape("example#Response$status", MemberShape).target + assert symbols.type_reference(target) == TypeReference(primitive, "builtins") + assert symbols.type_reference("example#Index") == TypeReference( + "dict", + "builtins", + ( + TypeReference("str", "builtins"), + TypeReference( + "list", + "builtins", + (TypeReference(primitive, "builtins", nullable=sparse),), + nullable=sparse, + ), + ), + ) + + def test_service_rename_only_changes_declaration() -> None: from smithy_python.selection import Selection from smithy_python.symbols import TypeReference @@ -113,9 +173,9 @@ def test_service_rename_only_changes_declaration() -> None: ) selection = select_shapes(model) symbols = SymbolProvider(model, selection, package="pkg") - assert symbols.declaration_name("example#Original") == "HttpServer" + assert symbols.declaration_name("example#Original") == "HTTPServer" assert symbols.type_reference("example#Original") == TypeReference( - "HttpServer", "pkg.models" + "HTTPServer", "pkg.models" ) without_service = SymbolProvider( model, Selection(None, selection.shapes, 0), package="pkg" @@ -248,19 +308,19 @@ def test_declaration_collisions_after_rename() -> None: "type": "service", "rename": {"example#Second": "HTTPServer"}, }, - "example#Http_Server": {"type": "structure"}, + "example#HTTP_Server": {"type": "structure"}, "example#Second": {"type": "enum"}, } ) selection = Selection( model.expect_shape("example#Service", ServiceShape), - (model.get_shape("example#Http_Server"), model.get_shape("example#Second")), + (model.get_shape("example#HTTP_Server"), model.get_shape("example#Second")), 0, ) with pytest.raises(ModelError) as error: SymbolProvider(model, selection, package="pkg") - assert "'HttpServer'" in str(error.value) - assert "example#Http_Server" in str(error.value) + assert "'HTTPServer'" in str(error.value) + assert "example#HTTP_Server" in str(error.value) assert "example#Second" in str(error.value) @@ -270,7 +330,7 @@ def test_only_emitted_declarations_collide() -> None: model = model_with( { "example#HTTPServer": {"type": "string"}, - "example#Http_Server": {"type": "structure"}, + "example#HTTP_Server": {"type": "structure"}, "example#Document": { "type": "structure", "members": { @@ -315,7 +375,7 @@ def test_invalid_package(package: str) -> None: @pytest.mark.parametrize( "name,expected", - [("2HTTPServer", "_2HttpServer"), ("__init__", "Init"), ("None", "None_")], + [("2HTTPServer", "_2HTTPServer"), ("__init__", "Init"), ("None", "None_")], ) def test_rename_normalization(name: str, expected: str) -> None: from smithy_python.model import ServiceShape From b1fba20d4d456453fd0c9c090f7117abe31bf11b Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sun, 27 Sep 2026 15:22:48 -0400 Subject: [PATCH 3/4] Allow digits after leading underscores Match Smithy identifier rules for shape names, members, and namespace segments. Keep rejecting names that start with a digit or contain only underscores. --- .../src/smithy_python/model/_shape_id.py | 2 +- .../smithy-python/tests/unit/model/test_shape_id.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/smithy-python/src/smithy_python/model/_shape_id.py b/packages/smithy-python/src/smithy_python/model/_shape_id.py index cb4fee272..60e6c39f8 100644 --- a/packages/smithy-python/src/smithy_python/model/_shape_id.py +++ b/packages/smithy-python/src/smithy_python/model/_shape_id.py @@ -10,7 +10,7 @@ from ..exceptions import InvalidShapeIdError -_IDENTIFIER: Final = r"_*[A-Za-z][A-Za-z0-9_]*" +_IDENTIFIER: Final = r"(?:[A-Za-z]|_+[A-Za-z0-9])[A-Za-z0-9_]*" _IDENTIFIER_RE: Final = re.compile(_IDENTIFIER) _NAMESPACE_RE: Final = re.compile(rf"{_IDENTIFIER}(?:\.{_IDENTIFIER})*") diff --git a/packages/smithy-python/tests/unit/model/test_shape_id.py b/packages/smithy-python/tests/unit/model/test_shape_id.py index ddd91c3bd..db0d90814 100644 --- a/packages/smithy-python/tests/unit/model/test_shape_id.py +++ b/packages/smithy-python/tests/unit/model/test_shape_id.py @@ -46,6 +46,10 @@ def test_relative_shape_id_without_namespace_fails() -> None: "a..b#Foo", "a.b#Foo$", "a.b#1Foo", + "2example#Foo", + "a.b#Foo$2bar", + "a.b#___", + "a.b#Foo$___", "a.b#Foo$m$n", "a#b#C", "a.b#Fo o", @@ -82,6 +86,14 @@ def test_shape_ids_are_hashable_and_ordered() -> None: assert ShapeId.from_string("a#B") == ShapeId("a", "B") +@pytest.mark.parametrize("name", ["_2HTTPServer", "__2", "_0"]) +def test_digits_after_leading_underscores(name: str) -> None: + value = f"{name}.example#{name}${name}" + shape_id = ShapeId.from_string(value) + assert shape_id == ShapeId(f"{name}.example", name, name) + assert str(shape_id) == value + + def test_underscore_identifiers_allowed() -> None: shape_id = ShapeId.from_string("a_b.c#_Foo$_x") assert shape_id.namespace == "a_b.c" From ff02b6f986e921a37ef37b97dbabc1ca8c1cd10c Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sun, 27 Sep 2026 15:36:25 -0400 Subject: [PATCH 4/4] Clarify symbol provider documentation Explain naming and type references with concrete examples. Cover valid underscore-prefixed numeric names in symbol tests. --- designs/codegen/symbols.md | 262 ++++++++++-------- .../smithy-python/tests/unit/test_symbols.py | 3 +- 2 files changed, 154 insertions(+), 111 deletions(-) diff --git a/designs/codegen/symbols.md b/designs/codegen/symbols.md index ecb49cbd6..ff812000b 100644 --- a/designs/codegen/symbols.md +++ b/designs/codegen/symbols.md @@ -1,84 +1,78 @@ -# Native Python Symbols - -`smithy_python.symbols` provides `SymbolProvider(model, selection, *, package)` and -frozen, hashable `TypeReference(name, module=None, arguments=(), nullable=False)`. -It uses only the standard library. Pass a loaded Model and its existing Selection; -it does not select shapes again, mutate either input, or generate files. - -## Contract - -* `declaration_name(id)` names selected structures, unions, enums and intEnums. - Only these supported declarations occupy `.models`. -* `member_name(member_id)` names structure/union fields or enum/intEnum constants. - List/map member names are not generated declarations and are rejected. -* `type_reference(id)` resolves a top-level data shape. To resolve a field's type, - pass its `MemberShape.target`, not its member ID. Requiredness, defaults and - field optionality are deliberately outside this API. - -IDs can be absolute strings or `ShapeId` values. Relative/malformed IDs raise -`InvalidShapeIdError`; missing shapes raise `ShapeNotFoundError`. Unselected -non-prelude shapes, control shapes, mixins, trait definitions, inappropriate -method requests and unsupported types raise `ModelError`. Prelude primitive -references and Unit work without selection. A manually narrowed Selection is -honored: references to omitted non-prelude targets fail rather than widening it. -The Model and Selection must describe the same loaded model. - -Package segments must be Python identifiers, not Python 3.12 hard keywords -(including their Unicode NFKC equivalents); invalid packages raise `ValueError`. -The supplied package spelling is preserved. Soft keywords are accepted. No -package is imported or checked for installation. +# Native Python symbols + +`SymbolProvider` answers three questions for the Python generator: + +* What should a generated class or enum be called? +* What should a field or enum constant be called? +* What Python type represents a value? + +Pass a loaded model, a selection made from that model, and the destination package: +`SymbolProvider(model, selection, *, package)`. The provider does not select more +shapes, change the model, import packages, or write files. It uses only the +standard library. + +## Names and types + +Use full Smithy IDs, such as `example#Response`, or equivalent `ShapeId` objects. +A `$` identifies a member inside a shape: `example#Response$statusCode` is the +`statusCode` field of `Response`. + +| Method | Question | Example result | +| --- | --- | --- | +| `declaration_name("example#Response")` | What do we call the generated definition? | `"Response"` | +| `member_name("example#Response$statusCode")` | What do we call this field? | `"status_code"` | +| `type_reference("smithy.api#Integer")` | What type of value does it hold? | `TypeReference("int", "builtins")` | + +`declaration_name` names structures, unions, enums and integer enums. Their +future definitions belong in `.models`. Primitive aliases, lists and +maps do not get separate definitions: a list of strings is `list[str]`, not a +new class. + +`member_name` names structure/union fields and enum constants. A list's internal +`member` and a map's `key` and `value` describe their contents, not Python fields, +so this method rejects them. A structure field that holds a list still has a +name: `Response$tags` can become `tags: list[str]`. + +To find a field's type, pass the shape it points to (`member.target`), not the +field's own ID (`member.id`). Whether the field is required, has a default, or +can be omitted is a separate decision for the future structure generator. ```python -from smithy_python.model import load_model +from smithy_python.model import MemberShape, load_model from smithy_python.selection import select_shapes from smithy_python.symbols import SymbolProvider, TypeReference model = load_model('''{ "smithy": "2.0", "shapes": { - "example#HTTPServer": { + "example#HTTPResponse": { "type": "structure", - "members": {"getURL": {"target": "smithy.api#String"}} - }, - "example#Servers": { - "type": "list", - "member": {"target": "example#HTTPServer"}, - "traits": {"smithy.api#sparse": {}} + "members": {"statusCode": {"target": "smithy.api#Integer"}} } } }''') symbols = SymbolProvider(model, select_shapes(model), package="example.client") -assert symbols.declaration_name("example#HTTPServer") == "HTTPServer" -assert symbols.member_name("example#HTTPServer$getURL") == "get_url" -assert symbols.type_reference("example#Servers") == TypeReference( - "list", "builtins", - (TypeReference("HTTPServer", "example.client.models", nullable=True),), -) +member = model.expect_shape("example#HTTPResponse$statusCode", MemberShape) + +assert symbols.declaration_name("example#HTTPResponse") == "HTTPResponse" +assert symbols.member_name(member.id) == "status_code" +assert symbols.type_reference(member.target) == TypeReference("int", "builtins") ``` -## Naming and collisions - -The selected service's rename is applied to declarations before normalization; -without a service the original shape name is used. IDs, member names in the -model, enum values and wire names are never changed. - -For declarations, split on underscores, uppercase the first character of each -nonempty part, and join without changing the remaining capitals. Existing -acronyms are preserved; lowercase words do not acquire invented acronyms. -For fields and enum constants, split an uppercase run before its final capital -when followed by lowercase; then split lowercase-or-digit followed by uppercase. -Join lowercase words with underscores for fields, or uppercase words for constants. -Empty words are discarded, including leading/trailing underscores. Prefix `_` if the result -starts with a digit. Append `_` for an exact Python 3.12 hard keyword, using a -frozen explicit list, not the interpreter's keyword module. Soft keywords -`match`, `case`, `type`, `_` are not reserved. Special Python names such as -`__init__`, `_name_`, and `__private` lose their underscore wrappers, avoiding -magic methods, enum sunder names and name mangling. Empty or non-ASCII-identifier -rename words fail with the original ID; leading-digit renames are supported. - -| Input | Declaration | Field | Enum constant | +## Naming rules + +Apply the selected service's rename first, if present. Otherwise use the shape's +original name. Python naming never changes Smithy IDs, wire names or enum values. + +For class and enum names, remove underscores and uppercase the first character +of each nonempty part, preserving its remaining capitals. Fields use snake_case; +enum constants use UPPER_SNAKE_CASE. Split words at lowercase-or-digit to uppercase +boundaries, and before the last capital of an uppercase run followed by lowercase. + +| Input | Class or enum | Field | Enum constant | | --- | --- | --- | --- | | HTTPServer | HTTPServer | http_server | HTTP_SERVER | +| http_server | HttpServer | http_server | HTTP_SERVER | | getURL | GetURL | get_url | GET_URL | | HTTP2Server | HTTP2Server | http2_server | HTTP2_SERVER | | getURL2Value | GetURL2Value | get_url2_value | GET_URL2_VALUE | @@ -88,60 +82,108 @@ rename words fail with the original ID; leading-digit renames are supported. | None | None_ | none | NONE | | match | Match | match | MATCH | -For a leading-digit rename, `2HTTPServer` becomes `_2HTTPServer`. +Discard empty underscore-separated parts, including leading/trailing underscores. +This avoids Python's special treatment of names such as `__init__` and `__private`. +Append `_` if the result is a reserved Python word. The reserved-word list is fixed +at Python 3.12 so results do not depend on the Python version running codegen. +Names such as `match`, `case` and `type` are valid Python field names and do not +need a trailing underscore. + +Rename values may contain only ASCII letters, digits and underscores, and must +contain at least one letter or digit. If removing leading underscores would leave +a digit at the start, keep one underscore: `_2HTTPServer` stays `_2HTTPServer`. + +When constructed, the provider checks names for every selected, supported +declaration and its members. Invalid renames fail here. Two declarations cannot +have the same Python name in the generated models module; two fields or constants +cannot have the same name within one declaration. A collision raises `ModelError` +with both Smithy IDs and the conflicting name. The first conflict follows model +selection and member order. No numbered suffixes are added to hide collisions. -Construction checks supported generated declarations in one module scope and -members in each separate declaration scope, after escaping. A collision raises -`ModelError` with both original IDs and the resulting Python name; the first -conflict follows selection and member order, independent of lookup order. -No numbering, builtin ban, speculative reservations or import-name collision -checks are performed. Primitive aliases and collections have no declarations. -Imported Document and generated Document retain different modules. +Builtin names such as `list` are allowed. References to identically named types +from different modules remain distinct; the future writer must handle import +aliases or qualified names. -## Type references and recursion +## Type references -`name` and `module` identify a type, `arguments` is an ordered tuple of nested -references, and `nullable=True` means this reference also permits None. The only -provider-produced reference without a module is `TypeReference("None")` for -Unit. There is no arbitrary metadata, import rendering or annotation-string API. +`TypeReference(name, module=None, arguments=(), nullable=False)` is an immutable, +hashable description of a type, not a Python annotation string. `name` and +`module` identify the type; `arguments` holds its element, key or value types. +`nullable=True` means the value can also be `None`. + +For example, `list[str]` is represented as: + +```python +TypeReference("list", "builtins", (TypeReference("str", "builtins"),)) +``` -| Smithy type | Symbolic Python type | +| Smithy type | Python value type | | --- | --- | -| boolean | builtins.bool | -| string | builtins.str | -| byte, short, integer, long, bigInteger | builtins.int | -| float, double | builtins.float | -| ordinary blob | builtins.bytes | +| boolean | bool | +| string, enum | str | +| byte, short, integer, long, bigInteger, intEnum | int | +| float, double | float | +| ordinary blob | bytes | | bigDecimal | decimal.Decimal | | timestamp | datetime.datetime | | document | smithy_core.documents.Document | | structure, union | `.models.` | -| enum | builtins.str | -| intEnum | builtins.int | -| list | builtins.list with one argument | -| map | builtins.dict with key and value arguments | +| list | list[T] | +| map | dict[K, V] | | smithy.api#Unit | None | -Enum declarations still have names and constants, but value references use -`str` or `int`, including inside collections, so annotations permit unknown -future values. For example, `declaration_name(Color)` returns `Color` while -`type_reference(Color)` returns `TypeReference("str", "builtins")`. This does -not implement runtime deserialization or validation. - -Ordinary primitive aliases resolve to their underlying type. Sparse lists mark -only the element reference nullable; sparse maps mark only the value reference -nullable, including nested collections. These module names are symbolic strings: -codegen never imports runtime packages. - -Named references terminate traversal; self/mutual recursion and recursion through -collections do not create cyclic symbol objects. Collection expansion uses an -iterative postorder worklist with per-call results, not the Python call stack. -Collection-only cycles raise `ModelError` with the cycle IDs, excluding any -noncyclic prefix. Failed lookups cannot -poison subsequent resolutions. Streaming blobs and unions are rejected when -resolved (including through collections), not treated as ordinary types. They -do not reserve declarations at construction. Resolving a named structure does -not inspect its fields; consumers must resolve field targets separately. - -No service/operation/resource symbols, writers, CLI integration, schemas, -serializers, dependency tracking, plugins, or runtime dependencies are added. +`T`, `K` and `V` stand for element, key and value types. References to Python's +built-in types record `"builtins"` as their module; generated annotations can +use the usual short names. + +Enums still have named declarations and constants, but their values use `str` or +`int` so fields can hold values added by the service in the future. For example, +`declaration_name("example#Color")` returns `"Color"`, while +`type_reference("example#Color")` returns `TypeReference("str", "builtins")`. +The same rule applies inside collections. Runtime validation and deserialization +are not implemented here. + +A sparse list permits `None` elements; a sparse map permits `None` values, not +keys. Each collection controls its own sparseness. An outer sparse list can +contain `None` instead of an inner list without allowing `None` inside that inner +list. The provider uses `nullable` only for these collection entries, not to +decide whether structure fields are optional. + +Module names are recorded without importing anything. `Unit`, which represents +no value, is the only result without a module: `TypeReference("None")`. + +## Recursion and unsupported shapes + +Structures and unions resolve to named references without expanding their +fields. This supports types such as `Node` containing `list[Node]`. Callers +resolve each field's target separately. + +Collection expansion does not use Python recursion. A cycle made entirely of +collections, such as two lists containing each other, raises `ModelError` with +the IDs in the cycle. A failed lookup does not affect later lookups. + +Streaming types are deferred until their Python interfaces are defined. The +provider rejects streaming blobs and event-stream unions rather than treating +them as ordinary values. + +## Inputs and errors + +The model and selection must come from the same load; the provider does not +check this precondition. Requests for shapes outside the selection fail rather +than silently adding them. Smithy's built-in types, such as `String`, `Integer` +and `Unit`, remain available without selecting them for generation. + +* Malformed or incomplete IDs raise `InvalidShapeIdError`. +* IDs absent from the model raise `ShapeNotFoundError`. +* Unsupported or unselected shapes and invalid method requests raise `ModelError`. + For example, `type_reference` requires a shape ID, not a member ID; + `member_name` requires a supported field or constant; `declaration_name` + requires a generated definition. Services, operations, resources, mixins and + trait definitions do not have data symbols. + +The package is a dotted Python name such as `example.client`. Each part must be +a valid Python identifier and cannot be a reserved word such as `class`. +Alternative Unicode spellings that Python converts to reserved words are also +rejected: `ｃｌａｓｓ` is treated as `class` for this check. Invalid packages raise +`ValueError`. Other accepted spellings are preserved. Names such as `match` are +allowed. The package need not exist or be installed. diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index e9496aaad..7a8edbbec 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -17,6 +17,7 @@ def model_with(shapes: dict[str, object]) -> Model: ("source", "declaration", "field", "constant"), [ ("HTTPServer", "HTTPServer", "http_server", "HTTP_SERVER"), + ("_2HTTPServer", "_2HTTPServer", "_2_http_server", "_2_HTTP_SERVER"), ("getURL", "GetURL", "get_url", "GET_URL"), ("HTTP_Server", "HTTPServer", "http_server", "HTTP_SERVER"), ("http_server", "HttpServer", "http_server", "HTTP_SERVER"), @@ -375,7 +376,7 @@ def test_invalid_package(package: str) -> None: @pytest.mark.parametrize( "name,expected", - [("2HTTPServer", "_2HTTPServer"), ("__init__", "Init"), ("None", "None_")], + [("_2HTTPServer", "_2HTTPServer"), ("__init__", "Init"), ("None", "None_")], ) def test_rename_normalization(name: str, expected: str) -> None: from smithy_python.model import ServiceShape