From 468df42e2318675fddf7b53a04cb942749cb9ca9 Mon Sep 17 00:00:00 2001 From: David Adrian Date: Mon, 9 Nov 2026 13:15:19 -0500 Subject: [PATCH] Add basic support for Python type annotations This prevents every ZSchema file from failing the Python typechecker and requiring various `#ignore` annotations. --- README.md | 6 +++- pyproject.toml | 10 ++++++- zschema/compounds.py | 9 +++--- zschema/fixtures/typing_schema.py | 35 +++++++++++++++++++++++ zschema/keys.py | 28 +++++++++++-------- zschema/leaves.py | 7 +++-- zschema/py.typed | 0 zschema/test_typing.py | 46 +++++++++++++++++++++++++++++++ 8 files changed, 120 insertions(+), 21 deletions(-) create mode 100644 zschema/fixtures/typing_schema.py create mode 100644 zschema/py.typed create mode 100644 zschema/test_typing.py diff --git a/README.md b/README.md index 2d76a66..246616a 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,11 @@ Similar to `doc`, fields can have a description, examples, units, min/max values, etc. A full list of attributes can be found here: https://github.com/zmap/zschema/blob/master/zschema/leaves.py#L25. +When writing schemas, Python typing support is the minimal required to allow +schema definitions to pass Python type checkers without requiring the file to +have the relevant `ignore` annotations. It does not statically validate every +option. Use `validate()` for document validation and to test schema compilation +and validity. Running Tests ============= @@ -192,7 +197,6 @@ pip3 install ".[tests]" pytest ``` - License and Copyright ===================== diff --git a/pyproject.toml b/pyproject.toml index 04a230a..b73c0c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,16 @@ dependencies = [ [project.optional-dependencies] tests = [ - "pytest" + "pytest", + "mypy", + "pyright", + "types-six", + "types-python-dateutil", + "types-pytz" ] [project.scripts] zschema = "zschema.__main__:main" + +[tool.setuptools.package-data] +zschema = ["py.typed"] diff --git a/zschema/compounds.py b/zschema/compounds.py index aae0811..2634091 100644 --- a/zschema/compounds.py +++ b/zschema/compounds.py @@ -5,6 +5,7 @@ import copy import json from collections import OrderedDict +from typing import Any, Dict, Optional from zschema.keys import Keyable, DataValidationException, MergeConflictException from zschema.keys import _NO_ARG @@ -27,7 +28,7 @@ def _proto_indent(string, n): # Track protobuf message definitions that have been emitted. -_proto_messages = OrderedDict() +_proto_messages: Dict[str, str] = OrderedDict() class ListOf(Keyable): @@ -162,9 +163,9 @@ def ListOfType( class SubRecord(Keyable): - DEFINITION = {} + DEFINITION: Dict[Any, Keyable] = {} ALLOW_UNKNOWN = False - TYPE_NAME = None + TYPE_NAME: Optional[str] = None ES_NESTED = False def __init__( @@ -517,7 +518,7 @@ def docs_bq(self, parent_category=None): class Record(SubRecord): VALIDATION_POLICY = "error" - ES_DYNAMIC_POLICY = None + ES_DYNAMIC_POLICY: Optional[str] = None def to_es(self, name): subrecord = SubRecord.to_es(self) diff --git a/zschema/fixtures/typing_schema.py b/zschema/fixtures/typing_schema.py new file mode 100644 index 0000000..4944562 --- /dev/null +++ b/zschema/fixtures/typing_schema.py @@ -0,0 +1,35 @@ +"""Representative consumer schema; checked without diagnostic suppressions.""" + +from zschema.compounds import ListOf, NestedListOf, Record, SubRecord, SubRecordType +from zschema.keys import Port +from zschema.leaves import Boolean, Enum, String, Unsigned32BitInteger + + +class NamedString(String): + DOC = "Inherited documentation" + REQUIRED = True + + +empty = SubRecord({}) +address = SubRecordType( + {"street": String()}, type_name="Address", required=True, doc="An address" +) +tag = String.with_args(doc="A tag", required=True) +schema: Record = Record( + { + "name": NamedString(required=False, doc=None), + "enabled": Boolean(required=True, examples=[True, False]), + "count": Unsigned32BitInteger(min_value=0, max_value=100, units="items"), + "state": Enum(values=["on", "off"]), + "address": address(doc="Home address"), + "tags": ListOf(tag(required=False), min_items=0, max_items=10), + "nested": NestedListOf(String(), "value"), + Port(443): SubRecord( + {"tls": Boolean()}, extends=empty, allow_unknown=True, es_nested=True + ), + }, + doc="Example schema", + validation_policy="error", + es_dynamic_policy="strict", +) +schema.validate({}, policy="error", path=["root", 0]) diff --git a/zschema/keys.py b/zschema/keys.py index e06a65b..6017bad 100644 --- a/zschema/keys.py +++ b/zschema/keys.py @@ -4,23 +4,27 @@ import logging import sys +from typing import Any, Dict, List, Optional, Set _keyable_counter = 0 -class _NO_ARG(object): +class _NoArg(object): __nonzero__ = lambda _: False __bool__ = lambda _: False def __new__(cls): if hasattr(cls, "_instance"): return cls._instance - retv = super(_NO_ARG, cls).__new__(cls) + retv = super(_NoArg, cls).__new__(cls) cls._instance = retv return retv -_NO_ARG = _NO_ARG() +# This sentinel means "use the class default", which is distinct from None. +# Keep it dynamically typed: otherwise typecheckers infer that unannotated +# parameters defaulting to it accept only the sentinel, not schema values. +_NO_ARG: Any = _NoArg() class Port(object): @@ -149,21 +153,21 @@ class Keyable(object): REQUIRED = False DEPRECATED = False DEPRECATED_TYPE = False - DOC = None - DESC = None - CATEGORY = None - EXAMPLES = [] - EXCLUDE = set([]) - METADATA = {} + DOC: Optional[str] = None + DESC: Optional[str] = None + CATEGORY: Optional[str] = None + EXAMPLES: List[Any] = [] + EXCLUDE: Set[str] = set([]) + METADATA: Dict[str, Any] = {} ALLOW_UNKNOWN = False VALIDATION_POLICY = "inherit" - EXPLICIT_INDEX = None + EXPLICIT_INDEX: Optional[int] = None PR_IGNORE = False # create a map from name of type to class. We can use this # in order to create the Python definition from JSON. We need # this in the web interface. We define this in keyable because - _types_by_name = {} + _types_by_name: Dict[str, Any] = {} def __or__(self, f): return f(self) @@ -409,4 +413,4 @@ class MergeConflictException(Exception): pass -_zschema_types_by_name = {} +_zschema_types_by_name: Dict[str, Any] = {} diff --git a/zschema/leaves.py b/zschema/leaves.py index 6889e68..e10d34f 100644 --- a/zschema/leaves.py +++ b/zschema/leaves.py @@ -9,6 +9,7 @@ import datetime import socket import pytz +from typing import Any, Optional from zschema.keys import Keyable, DataValidationException from zschema.keys import _NO_ARG @@ -19,8 +20,8 @@ class Leaf(Keyable): # defaults REQUIRED = False ES_INCLUDE_RAW = False - ES_INDEX = None - ES_ANALYZER = None + ES_INDEX: Optional[str] = None + ES_ANALYZER: Optional[str] = None def __init__( self, @@ -199,7 +200,7 @@ class String(Leaf): EXPECTED_CLASS = string_types - INVALID = 23 + INVALID: Any = 23 VALID = "asdf" diff --git a/zschema/py.typed b/zschema/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/zschema/test_typing.py b/zschema/test_typing.py new file mode 100644 index 0000000..590b0b3 --- /dev/null +++ b/zschema/test_typing.py @@ -0,0 +1,46 @@ +"""Check schema consumers with both mainstream typecheckers.""" + +import copy +from pathlib import Path +import subprocess +import sys + +import pytest + +from zschema.keys import _NO_ARG +from zschema.leaves import String + + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("checker", ["mypy", "pyright"]) +def test_schema_typing(checker): + result = subprocess.run( + [ + sys.executable, + "-m", + checker, + "zschema/fixtures/typing_schema.py", + "zschema/example.py", + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_sentinel_preserves_defaults(): + class DefaultString(String): + DOC = "Inherited documentation" + REQUIRED = True + + assert not _NO_ARG + assert copy.copy(_NO_ARG) is _NO_ARG + assert copy.deepcopy(_NO_ARG) is _NO_ARG + assert DefaultString().doc == "Inherited documentation" + assert DefaultString().required is True + assert DefaultString(doc=None).doc is None + assert DefaultString(required=False).required is False + assert copy.deepcopy(DefaultString()).doc == "Inherited documentation"