diff --git a/mypyc/doc/native_classes.rst b/mypyc/doc/native_classes.rst index f7d91808d912..a97e4d39cbca 100644 --- a/mypyc/doc/native_classes.rst +++ b/mypyc/doc/native_classes.rst @@ -66,6 +66,12 @@ classes: * ``LookupError`` * ``UserWarning`` +A native class can't inherit from a base class with type ``Any``, such +as a class imported from a module that isn't type checked, even +indirectly through other base classes, since mypyc needs to know the +layout of all base classes. Use a +:ref:`non-native class ` instead. + By default, a non-native class can't inherit a native class, and you can't inherit from a native class outside the compilation unit that defines the class. You can enable these through @@ -231,6 +237,8 @@ efficient as pure native classes. If a class definition uses an unsupported class decorator, *mypyc compiles the class into a regular Python class* (non-native class). +.. _non-native-classes: + Defining non-native classes --------------------------- diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index ae63ff90c137..a0aa7c4ac26f 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -599,14 +599,16 @@ def uses_non_ext_base_exprs(cdef: ClassDef) -> bool: """Should the base classes of a non-extension class be evaluated as written? Resolved base types erase Generic[...] bases and type arguments, but they are - needed at runtime, e.g. to make the class generic. + needed at runtime, e.g. to make the class generic. Any bases are missing from + resolved base types, so Python must determine the bases and instance layout. """ info = cdef.info if info.is_named_tuple or info.typeddict_type is not None: # These have synthesized base classes that need special casing. return False return bool( - cdef.type_args + info.fallback_to_any + or cdef.type_args or cdef.removed_base_type_exprs or any(isinstance(expr, IndexExpr) for expr in cdef.base_type_exprs) ) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 318f94a4bf5f..be806b7f8b0d 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -34,6 +34,7 @@ OverloadedFuncDef, RefExpr, SymbolNode, + TypeAlias, TypeInfo, Var, ) @@ -539,6 +540,48 @@ def _check_deletable_declarations(path: str, cdef: ClassDef, ir: ClassIR, errors ) +def report_needs_non_native_class(errors: Errors, msg: str, path: str, line: int) -> None: + errors.error(msg, path, line) + errors.note( + "Potential workaround: @mypy_extensions.mypyc_attr(native_class=False)", path, line + ) + errors.note( + "https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes", + path, + line, + ) + + +def check_unknown_base_classes(path: str, cdef: ClassDef, errors: Errors, mapper: Mapper) -> None: + """Reject native classes whose layout depends on an Any base.""" + if not cdef.info.fallback_to_any: + return + + base_description = "a base class with an unknown layout" + for base_expr in cdef.base_type_exprs: + if isinstance(base_expr, IndexExpr): + base_expr = base_expr.base + if not isinstance(base_expr, (NameExpr, MemberExpr)): + continue + base_node = base_expr.node + if isinstance(base_node, TypeAlias): + target = get_proper_type(base_node.target) + if isinstance(target, Instance): + base_node = target.type + if not isinstance(base_node, TypeInfo) or base_node.fallback_to_any: + base_description = f'"{base_expr.name}" (unknown base class layout)' + break + msg = f"Native classes cannot inherit from {base_description}" + if any( + base.fallback_to_any and base in mapper.type_to_ir and mapper.type_to_ir[base].is_ext_class + for base in cdef.info.mro[1:] + ): + # A native base class already reported the error with the notes + errors.error(msg, path, cdef.line) + else: + report_needs_non_native_class(errors, msg, path, cdef.line) + + def prepare_class_def( path: str, module_name: str, @@ -556,6 +599,8 @@ def prepare_class_def( ir = mapper.type_to_ir[cdef.info] info = cdef.info + check_unknown_base_classes(path, cdef, errors, mapper) + attrs, attrs_lines = get_mypyc_attrs(cdef, path, errors) if attrs.get("allow_interpreted_subclasses") is True: ir.allow_interpreted_subclasses = True @@ -580,18 +625,8 @@ def prepare_class_def( # isn't in builtins, bad things will happen and we won't # catch it here! But this should catch a lot of the most # common pitfalls. - errors.error( - "Inheriting from most builtin types is unimplemented", path, cdef.line - ) - errors.note( - "Potential workaround: @mypy_extensions.mypyc_attr(native_class=False)", - path, - cdef.line, - ) - errors.note( - "https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes", - path, - cdef.line, + report_needs_non_native_class( + errors, "Inheriting from most builtin types is unimplemented", path, cdef.line ) free_list_len = attrs.get("free_list_len") diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 1d2343ca5917..6df6c6221679 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1718,6 +1718,90 @@ class NonNativeClassContradiction(): # E: Class is marked as native_class=True class BadUse(): # E: native_class must be used with True or False only pass +[case testNativeClassUnknownBase] +from mypy_extensions import mypyc_attr +from typing import Any +from missing_module import Base # type: ignore + +class C(Base): # E: Native classes cannot inherit from "Base" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class D(C): # E: Native classes cannot inherit from "C" (unknown base class layout) + pass + +@mypyc_attr(native_class=False) +class NonNative(Base): + pass + +class NativeChild(NonNative): # E: Native classes cannot inherit from "NonNative" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +ExplicitAny: Any = str +class S(ExplicitAny): # E: Native classes cannot inherit from "ExplicitAny" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class Known: + pass + +class KnownFirst(Known, Base): # E: Native classes cannot inherit from "Base" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class UnknownFirst(Base, Known): # E: Native classes cannot inherit from "Base" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class TwoUnknown(Base, ExplicitAny): # E: Native classes cannot inherit from "Base" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +Alias = Known +class AliasFirst(Alias, Base): # E: Native classes cannot inherit from "Base" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class Indexed(Base[int]): # type: ignore # E: Native classes cannot inherit from "Base" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +def make_base() -> Any: ... +class Dynamic(make_base()): # type: ignore # E: Native classes cannot inherit from a base class with an unknown layout \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +[case testNativeClassIndirectUnknownBase] +# Any base class anywhere in the MRO makes the layout unknown +from stubbed import Indirect, IndirectAlias + +class C(Indirect): # E: Native classes cannot inherit from "Indirect" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class D(IndirectAlias): # E: Native classes cannot inherit from "IndirectAlias" (unknown base class layout) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +[file stubbed.pyi] +from typing import Any + +class AnyBase(Any): ... +class Indirect(AnyBase): ... +IndirectAlias = Indirect + [case testMypycAttrNativeClassMetaError] from mypy_extensions import mypyc_attr diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index ebeb04cf1fa5..d7a0a0a334ae 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -1543,6 +1543,161 @@ def test_raise_non_native_exception_subclass_2() -> None: assert False assert x +[case testSubclassUnknownPythonBase] +# The base class is unknown during compilation, so it has type Any +from mypy_extensions import mypyc_attr +from typing import Any, Generic, TypeVar +from unchecked import Base, MetaBase, SecondBase, StrBase # type: ignore + +T = TypeVar("T") + +@mypyc_attr(native_class=False) +class C(Base): + def __init__(self, x: int) -> None: + super().__init__() + self.x = x + + def marker(self) -> str: + return "override " + super().marker() + +@mypyc_attr(native_class=False) +class InheritedInit(Base): + pass + +@mypyc_attr(native_class=False) +class D(InheritedInit): + pass + +@mypyc_attr(native_class=False) +class S(StrBase): + pass + +@mypyc_attr(native_class=False) +class WithMeta(MetaBase): + pass + +@mypyc_attr(native_class=False) +class Known: + def marker(self) -> str: + return "known" + +@mypyc_attr(native_class=False) +class KnownFirst(Known, Base): + pass + +@mypyc_attr(native_class=False) +class UnknownFirst(Base, Known): + pass + +@mypyc_attr(native_class=False) +class TwoUnknown(Base, SecondBase): + pass + +@mypyc_attr(native_class=False) +class GenericLast(Base, Generic[T]): + pass + +@mypyc_attr(native_class=False) +class GenericFirst(Generic[T], Base): + pass + +@mypyc_attr(native_class=False) +class KnownGeneric(Generic[T]): + pass + +@mypyc_attr(native_class=False) +class KnownGenericBase(Base, KnownGeneric[int]): + pass + +def test_unknown_base() -> None: + c = C(1) + assert c.x == 1 + assert c.y == 2 + assert c.label == "base attribute" + assert c.marker() == "override base" + c.z = 3 + assert c.z == 3 + assert c.__dict__ == {"x": 1, "y": 2, "z": 3} + assert isinstance(c, Base) + +def test_inherited_init() -> None: + assert InheritedInit().y == 2 + assert InheritedInit(7).y == 7 + assert InheritedInit(x=11).y == 11 + d = D(13) + assert d.y == 13 + assert d.label == "base attribute" + assert d.marker() == "base" + assert D(x=17).y == 17 + try: + InheritedInit(-1) + except ValueError as e: + assert str(e) == "negative" + else: + assert False + +def test_unknown_extension_base() -> None: + assert S.__basicsize__ >= StrBase.__basicsize__ + c = S("abc") + assert isinstance(c, str) + assert c == "abc" + assert len(c) == 3 + assert c.upper() == "ABC" + +def test_unknown_base_metaclass() -> None: + assert isinstance(WithMeta, type(MetaBase)) + assert isinstance(WithMeta(), MetaBase) + +def test_multiple_inheritance() -> None: + assert KnownFirst.__bases__ == (Known, Base) + assert UnknownFirst.__bases__ == (Base, Known) + assert TwoUnknown.__bases__ == (Base, SecondBase) + assert KnownFirst(3).y == 3 + assert UnknownFirst(x=5).y == 5 + assert KnownFirst().marker() == "known" + assert UnknownFirst().marker() == "base" + assert TwoUnknown(7).y == 7 + assert TwoUnknown().other() == "second" + +def test_generic_bases() -> None: + generic: Any = globals()["Generic"] + type_var: Any = globals()["T"] + assert GenericLast.__bases__ == (Base, generic) + assert GenericFirst.__bases__ == (generic, Base) + assert GenericLast.__orig_bases__ == (Base, generic[type_var]) + assert GenericFirst.__orig_bases__ == (generic[type_var], Base) + assert GenericLast.__parameters__ == (type_var,) + assert GenericFirst.__parameters__ == (type_var,) + assert GenericLast[int](3).y == 3 + assert GenericFirst[str](5).y == 5 + assert KnownGenericBase.__bases__ == (Base, KnownGeneric) + assert KnownGenericBase.__orig_bases__ == (Base, KnownGeneric[int]) + assert KnownGenericBase(7).y == 7 + +[file unchecked.py] +StrBase = str + +class SecondBase: + def other(self): + return "second" + +class Meta(type): + pass + +class MetaBase(metaclass=Meta): + pass + +class Base: + label = "base attribute" + + def __init__(self, x=2): + if x < 0: + raise ValueError("negative") + self.y = x + + def marker(self): + return "base" + [case testSubclassPy] from b import B, V class A(B): @@ -2066,26 +2221,30 @@ native.test() [out] ok -[case testNoMetaclass] -from foo import Base +[case testNativeClassHiddenMetaclass] +# The stub hides the metaclass of the base class, so it's only detected at runtime +from hidden import Base -class Nothing(Base): # type: ignore +class C(Base): pass -[file foo.py] -from typing import Any +[file hidden.pyi] +class Base: ... + +[file hidden.py] class Meta(type): pass -class _Base(metaclass=Meta): +class Base(metaclass=Meta): pass -Base = _Base # type: Any [file driver.py] try: import native except TypeError as e: - assert(str(e) == "mypyc classes can't have a metaclass") + assert str(e) == "mypyc classes can't have a metaclass" +else: + assert False [case testMetaclass] from meta import Meta