Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ Similar to `doc`, fields can have a description, examples, units, min/max
values, etc. A full list of attributes can be found here:
https://github.com/zmap/zschema/blob/master/zschema/leaves.py#L25.

When writing schemas, Python typing support is the minimal required to allow
schema definitions to pass Python type checkers without requiring the file to
have the relevant `ignore` annotations. It does not statically validate every
option. Use `validate()` for document validation and to test schema compilation
and validity.

Running Tests
=============
Expand All @@ -192,7 +197,6 @@ pip3 install ".[tests]"
pytest
```


License and Copyright
=====================

Expand Down
10 changes: 9 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,16 @@ dependencies = [

[project.optional-dependencies]
tests = [
"pytest"
"pytest",
"mypy",
"pyright",
"types-six",
"types-python-dateutil",
"types-pytz"
]

[project.scripts]
zschema = "zschema.__main__:main"

[tool.setuptools.package-data]
zschema = ["py.typed"]
9 changes: 5 additions & 4 deletions zschema/compounds.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import copy
import json
from collections import OrderedDict
from typing import Any, Dict, Optional

from zschema.keys import Keyable, DataValidationException, MergeConflictException
from zschema.keys import _NO_ARG
Expand All @@ -27,7 +28,7 @@ def _proto_indent(string, n):


# Track protobuf message definitions that have been emitted.
_proto_messages = OrderedDict()
_proto_messages: Dict[str, str] = OrderedDict()


class ListOf(Keyable):
Expand Down Expand Up @@ -162,9 +163,9 @@ def ListOfType(

class SubRecord(Keyable):

DEFINITION = {}
DEFINITION: Dict[Any, Keyable] = {}
ALLOW_UNKNOWN = False
TYPE_NAME = None
TYPE_NAME: Optional[str] = None
ES_NESTED = False

def __init__(
Expand Down Expand Up @@ -517,7 +518,7 @@ def docs_bq(self, parent_category=None):
class Record(SubRecord):

VALIDATION_POLICY = "error"
ES_DYNAMIC_POLICY = None
ES_DYNAMIC_POLICY: Optional[str] = None

def to_es(self, name):
subrecord = SubRecord.to_es(self)
Expand Down
35 changes: 35 additions & 0 deletions zschema/fixtures/typing_schema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""Representative consumer schema; checked without diagnostic suppressions."""

from zschema.compounds import ListOf, NestedListOf, Record, SubRecord, SubRecordType
from zschema.keys import Port
from zschema.leaves import Boolean, Enum, String, Unsigned32BitInteger


class NamedString(String):
DOC = "Inherited documentation"
REQUIRED = True


empty = SubRecord({})
address = SubRecordType(
{"street": String()}, type_name="Address", required=True, doc="An address"
)
tag = String.with_args(doc="A tag", required=True)
schema: Record = Record(
{
"name": NamedString(required=False, doc=None),
"enabled": Boolean(required=True, examples=[True, False]),
"count": Unsigned32BitInteger(min_value=0, max_value=100, units="items"),
"state": Enum(values=["on", "off"]),
"address": address(doc="Home address"),
"tags": ListOf(tag(required=False), min_items=0, max_items=10),
"nested": NestedListOf(String(), "value"),
Port(443): SubRecord(
{"tls": Boolean()}, extends=empty, allow_unknown=True, es_nested=True
),
},
doc="Example schema",
validation_policy="error",
es_dynamic_policy="strict",
)
schema.validate({}, policy="error", path=["root", 0])
28 changes: 16 additions & 12 deletions zschema/keys.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,27 @@

import logging
import sys
from typing import Any, Dict, List, Optional, Set

_keyable_counter = 0


class _NO_ARG(object):
class _NoArg(object):
__nonzero__ = lambda _: False
__bool__ = lambda _: False

def __new__(cls):
if hasattr(cls, "_instance"):
return cls._instance
retv = super(_NO_ARG, cls).__new__(cls)
retv = super(_NoArg, cls).__new__(cls)
cls._instance = retv
return retv


_NO_ARG = _NO_ARG()
# This sentinel means "use the class default", which is distinct from None.
# Keep it dynamically typed: otherwise typecheckers infer that unannotated
# parameters defaulting to it accept only the sentinel, not schema values.
_NO_ARG: Any = _NoArg()


class Port(object):
Expand Down Expand Up @@ -149,21 +153,21 @@ class Keyable(object):
REQUIRED = False
DEPRECATED = False
DEPRECATED_TYPE = False
DOC = None
DESC = None
CATEGORY = None
EXAMPLES = []
EXCLUDE = set([])
METADATA = {}
DOC: Optional[str] = None
DESC: Optional[str] = None
CATEGORY: Optional[str] = None
EXAMPLES: List[Any] = []
EXCLUDE: Set[str] = set([])
METADATA: Dict[str, Any] = {}
ALLOW_UNKNOWN = False
VALIDATION_POLICY = "inherit"
EXPLICIT_INDEX = None
EXPLICIT_INDEX: Optional[int] = None
PR_IGNORE = False

# create a map from name of type to class. We can use this
# in order to create the Python definition from JSON. We need
# this in the web interface. We define this in keyable because
_types_by_name = {}
_types_by_name: Dict[str, Any] = {}

def __or__(self, f):
return f(self)
Expand Down Expand Up @@ -409,4 +413,4 @@ class MergeConflictException(Exception):
pass


_zschema_types_by_name = {}
_zschema_types_by_name: Dict[str, Any] = {}
7 changes: 4 additions & 3 deletions zschema/leaves.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import datetime
import socket
import pytz
from typing import Any, Optional

from zschema.keys import Keyable, DataValidationException
from zschema.keys import _NO_ARG
Expand All @@ -19,8 +20,8 @@ class Leaf(Keyable):
# defaults
REQUIRED = False
ES_INCLUDE_RAW = False
ES_INDEX = None
ES_ANALYZER = None
ES_INDEX: Optional[str] = None
ES_ANALYZER: Optional[str] = None

def __init__(
self,
Expand Down Expand Up @@ -199,7 +200,7 @@ class String(Leaf):

EXPECTED_CLASS = string_types

INVALID = 23
INVALID: Any = 23
VALID = "asdf"


Expand Down
Empty file added zschema/py.typed
Empty file.
46 changes: 46 additions & 0 deletions zschema/test_typing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
"""Check schema consumers with both mainstream typecheckers."""

import copy
from pathlib import Path
import subprocess
import sys

import pytest

from zschema.keys import _NO_ARG
from zschema.leaves import String


ROOT = Path(__file__).resolve().parents[1]


@pytest.mark.parametrize("checker", ["mypy", "pyright"])
def test_schema_typing(checker):
result = subprocess.run(
[
sys.executable,
"-m",
checker,
"zschema/fixtures/typing_schema.py",
"zschema/example.py",
],
cwd=ROOT,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stdout + result.stderr


def test_sentinel_preserves_defaults():
class DefaultString(String):
DOC = "Inherited documentation"
REQUIRED = True

assert not _NO_ARG
assert copy.copy(_NO_ARG) is _NO_ARG
assert copy.deepcopy(_NO_ARG) is _NO_ARG
assert DefaultString().doc == "Inherited documentation"
assert DefaultString().required is True
assert DefaultString(doc=None).doc is None
assert DefaultString(required=False).required is False
assert copy.deepcopy(DefaultString()).doc == "Inherited documentation"
Loading