diff --git a/docs/source/contributing/conventions/reassembly-integration.rst b/docs/source/contributing/conventions/reassembly-integration.rst index 75fcf67fdc..fdcbf68863 100644 --- a/docs/source/contributing/conventions/reassembly-integration.rst +++ b/docs/source/contributing/conventions/reassembly-integration.rst @@ -62,10 +62,12 @@ the default engine and is disabled at the engine level on every other. handshake message (:issue:`2496`). IP and TCP have no such method: what parses their outcome is the general-purpose :meth:`Protocol.analyze ` dispatcher, -reused via :class:`~pcapkit.foundation.reassembly.data.data.Deferred`, not a method -written for reassembly specifically. +not a method written for reassembly specifically. -DTLS differs from the IP/TCP shape where the rule is silent: its reassembled body is -reported eagerly, with no deferred ``packet``, and -:class:`~pcapkit.foundation.reassembly.dtls.DTLS` sets no ``__protocol_type__`` -(:issue:`2531`). +**Where the rule is silent, DTLS takes the IP/TCP shape** (:issue:`2531`). IPv4, IPv6, +TCP and DTLS reassembly each set ``__protocol_type__``, and each reports its parsed +outcome as a datagram's ``packet``, deferred to first read by +:class:`~pcapkit.foundation.reassembly.data.data.Deferred`. IP and TCP run +``__protocol_type__``'s ``analyze``. DTLS runs the decoder behind ``dtls_handshake`` +instead, where point 3 places it, so its ``packet`` is what ``dtls_handshake`` returns: +:data:`None` for an incomplete message. diff --git a/docs/source/pcapkit/foundation/reassembly/dtls.rst b/docs/source/pcapkit/foundation/reassembly/dtls.rst index ec7ff2ed23..872a6f33ab 100644 --- a/docs/source/pcapkit/foundation/reassembly/dtls.rst +++ b/docs/source/pcapkit/foundation/reassembly/dtls.rst @@ -22,6 +22,7 @@ DTLS handshake message split across several records back to origin .. automethod:: _update_hole_descriptors .. autoattribute:: __protocol_name__ + .. autoattribute:: __protocol_type__ Algorithm ========= @@ -63,10 +64,12 @@ Terminology :attr:`DTLS.datagram `) is :class:`~pcapkit.foundation.reassembly.data.dtls.Datagram` -- ``completed``, the original packet identifier, the packet numbers - that built it, and ``body``, the reassembled payload (a contiguous + that built it, ``body``, the reassembled payload (a contiguous :obj:`bytes`, or -- under ``strict=True`` and an incomplete message -- - a :obj:`tuple` of the octet runs actually received), which - :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` decodes. + a :obj:`tuple` of the octet runs actually received), and ``packet``, + what :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` decodes + ``body`` to. ``packet`` is decoded on first read, as an IP or TCP + datagram's is, and is :data:`None` for an incomplete message. reasm.dtls.buffer Data structure for internal buffering diff --git a/pcapkit/corekit/packet.py b/pcapkit/corekit/packet.py index f47dff8ec8..999f8408f7 100644 --- a/pcapkit/corekit/packet.py +++ b/pcapkit/corekit/packet.py @@ -61,7 +61,8 @@ class DeferredPacket: It is shared by :class:`pcapkit.foundation.reassembly.data.ip.Datagram`, - :class:`pcapkit.foundation.reassembly.data.tcp.Datagram` and + :class:`pcapkit.foundation.reassembly.data.tcp.Datagram`, + :class:`pcapkit.foundation.reassembly.data.dtls.Datagram` and :class:`pcapkit.foundation.traceflow.data.tcp.Index`, which differ only in the placeholder class each names in :attr:`__deferred__` (:issue:`1516`). diff --git a/pcapkit/foundation/analysis/dtls.py b/pcapkit/foundation/analysis/dtls.py index ff23176976..47487ec547 100644 --- a/pcapkit/foundation/analysis/dtls.py +++ b/pcapkit/foundation/analysis/dtls.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from typing import Optional + from pcapkit.const.tls.handshake_type import HandshakeType as Enum_HandshakeType from pcapkit.foundation.reassembly.data.dtls import Datagram as DTLS_Datagram __all__ = ['dtls_handshake'] @@ -45,23 +46,47 @@ def dtls_handshake(datagram: 'DTLS_Datagram') -> 'Optional[Data_DTLSHandshake]': they do not decode as its ``msg_type``. :data:`None` if ``datagram`` is incomplete, as its missing octets cannot be decoded. + Note: + :attr:`Datagram.packet ` + holds the same result, decoded on its first read. + """ if not datagram.completed: return None # NOTE: a complete message is always one contiguous run. - body = cast('bytes', datagram.body) - message = (bytes([int(datagram.msg_type)]) + datagram.length.to_bytes(3, 'big') - + datagram.message_seq.to_bytes(2, 'big') + bytes(3) # fragment_offset 0 - + datagram.length.to_bytes(3, 'big') + body) + return _read_handshake((datagram.msg_type, datagram.length, datagram.message_seq), + cast('bytes', datagram.body)) + + +def _read_handshake(header: 'tuple[Enum_HandshakeType, int, int]', body: 'bytes') -> 'Data_DTLSHandshake': + """Decode a complete handshake message from its header and body. + + :func:`dtls_handshake` runs this, and so does the + :class:`~pcapkit.foundation.reassembly.data.data.Deferred` that + :class:`~pcapkit.foundation.reassembly.dtls.DTLS` stores as a complete + message's ``packet``. + + Args: + header: The message's ``(msg_type, length, message_seq)``. + body: The reassembled body. + + Returns: + The message, unfragmented, its body raw if it does not decode. + + """ + msg_type, length, message_seq = header + message = (bytes([int(msg_type)]) + length.to_bytes(3, 'big') + + message_seq.to_bytes(2, 'big') + bytes(3) # fragment_offset 0 + + length.to_bytes(3, 'big') + body) try: return DTLS.read_handshake(message) except ProtocolError: return Data_DTLSHandshake( - msg_type=datagram.msg_type, - length=datagram.length, + msg_type=msg_type, + length=length, body=body, - message_seq=datagram.message_seq, + message_seq=message_seq, fragment_offset=0, - fragment_length=datagram.length, + fragment_length=length, is_fragmented=False, ) diff --git a/pcapkit/foundation/reassembly/data/data.py b/pcapkit/foundation/reassembly/data/data.py index 274adc1fce..828258ec12 100644 --- a/pcapkit/foundation/reassembly/data/data.py +++ b/pcapkit/foundation/reassembly/data/data.py @@ -15,13 +15,15 @@ from pcapkit.const.reg.apptype import TCP as AppType_TCP from pcapkit.const.reg.transtype import TransType + from pcapkit.const.tls.handshake_type import HandshakeType from pcapkit.foundation.reassembly.data.dtls import Datagram as DTLS_Datagram from pcapkit.foundation.reassembly.data.ip import Datagram as IP_Datagram from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram + from pcapkit.protocols.data.application.dtls import DTLSHandshake from pcapkit.protocols.protocol import ProtocolBase # registry key an analyser dispatches a reassembled payload on; see ``Deferred`` -_KT = TypeVar('_KT', 'TransType', 'tuple[AppType_TCP, AppType_TCP]') +_KT = TypeVar('_KT', 'TransType', 'tuple[AppType_TCP, AppType_TCP]', 'tuple[HandshakeType, int, int]') class Completion(EnumLookup, StrEnum): @@ -125,7 +127,7 @@ class Deferred(Generic[_KT]): the datagram just reassembled. Nothing about postponing it is specific to any one reassembler, which is why this lives beside :class:`~pcapkit.foundation.reassembly.data.data.ReassemblyData` rather than in - either protocol's data module. + any one protocol's data module. IP reassembly is the case that made it necessary. IPv4 reassembly submits a datagram for *every* frame whose **DF** flag is clear -- not only the @@ -139,7 +141,9 @@ class Deferred(Generic[_KT]): TCP reassembly defers its ``packet`` as well (:meth:`TCP.submit `), at a far smaller saving, being FIN/RST-driven rather than per-frame -- 222 submits per - :file:`http.pcap` pass against 1117. + :file:`http.pcap` pass against 1117. DTLS reassembly defers its own for + parity with IP and TCP rather than for a saving, since a handshake message + reaches it only once fragmented (:issue:`2531`). Holding the call here defers it to the first read of :attr:`Datagram.packet `, @@ -147,10 +151,10 @@ class Deferred(Generic[_KT]): wants the parsed payload still gets exactly the object an eager call would produce, and one that does not never pays for it. - The class is generic because the two reassemblers do *not* dispatch on the + The class is generic because the reassemblers do *not* dispatch on the same kind of value, and each analyser declares the one it takes. The type parameter is that value's type, so an instance's ``proto`` and the parameter - its ``analyze`` expects are the same type rather than a union either call + its ``analyze`` expects are the same type rather than a union every call site would have to widen to: * :meth:`IP.submit ` builds a @@ -165,8 +169,15 @@ class Deferred(Generic[_KT]): ` overrides that parameter to one: a reassembled transport-layer payload carries no protocol number, so its application protocol is inferred from the ports. - - Those are the two cases the type parameter is constrained to. Both are + * :meth:`DTLS._datagram_from_fragment + ` builds a + ``Deferred[tuple[HandshakeType, int, int]]`` for a complete message. It + passes the message's ``(msg_type, length, message_seq)``, its handshake + header as if unfragmented (:rfc:`6347#section-4.2.3`), to the decoder + :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` runs, which reads + the body by its :class:`~pcapkit.const.tls.handshake_type.HandshakeType`. + + Those are the cases the type parameter is constrained to. IP's and TCP's are *named registry values* rather than one named value and one pair of bare port numbers: a port is a member of :class:`~pcapkit.const.reg.apptype.tcp.TCP` here -- the per-transport @@ -198,20 +209,28 @@ class Deferred(Generic[_KT]): either way. Args: - analyze: The analyser to call, i.e. + analyze: The analyser to call: :meth:`Protocol.analyze ` - bound to the reassembly object's protocol. + bound to the reassembly object's protocol for IP and TCP, the + handshake decoder for DTLS. proto: What the analyser dispatches the payload on -- the payload - protocol type for IP, the port pair for TCP, as above. + protocol type for IP, the port pair for TCP, the handshake header + for DTLS, as above. payload: Reassembled payload to parse. .. versionadded:: 1.5.0 + .. versionchanged:: 1.6.0 + + Also constrained to the DTLS handshake header, and the analyser may + return a :class:`~pcapkit.protocols.data.application.dtls.DTLSHandshake` + (:issue:`2531`). + """ __slots__ = ('analyze', 'proto', 'payload') - def __init__(self, analyze: 'Callable[[_KT, bytes], ProtocolBase]', + def __init__(self, analyze: 'Callable[[_KT, bytes], ProtocolBase | DTLSHandshake]', proto: '_KT', payload: 'bytes') -> 'None': # NOTE: the two type-parameterized attributes are annotated on their # assignment, as ``_omd_bucket`` in :mod:`pcapkit.corekit.multidict` @@ -219,11 +238,11 @@ def __init__(self, analyze: 'Callable[[_KT, bytes], ProtocolBase]', # infers each one from this assignment, and it cannot infer a *class* # attribute's type from a type variable -- it asks for an annotation # instead. ``payload`` needs none: :obj:`bytes` is inferred. - self.analyze: 'Callable[[_KT, bytes], ProtocolBase]' = analyze + self.analyze: 'Callable[[_KT, bytes], ProtocolBase | DTLSHandshake]' = analyze self.proto: '_KT' = proto self.payload = payload - def __call__(self) -> 'ProtocolBase': + def __call__(self) -> 'ProtocolBase | DTLSHandshake': """Run the postponed analysis. Returns: diff --git a/pcapkit/foundation/reassembly/data/dtls.py b/pcapkit/foundation/reassembly/data/dtls.py index 6716905f41..75a8728938 100644 --- a/pcapkit/foundation/reassembly/data/dtls.py +++ b/pcapkit/foundation/reassembly/data/dtls.py @@ -4,6 +4,8 @@ from typing import TYPE_CHECKING, Generic, TypeVar from pcapkit.corekit.infoclass import Info, info_final +from pcapkit.corekit.packet import DeferredPacket +from pcapkit.foundation.reassembly.data.data import Deferred from pcapkit.utilities.compat import Tuple __all__ = [ @@ -18,6 +20,7 @@ from pcapkit.const.tls.handshake_type import HandshakeType as Enum_HandshakeType from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.protocols.data.application.dtls import DTLSHandshake as Data_DTLSHandshake _AT = TypeVar('_AT', 'IPv4Address', 'IPv6Address') @@ -81,17 +84,16 @@ def __init__(self, src: '_AT', sport: 'int', dst: '_AT', dport: 'int', epoch: 'i @info_final -class Datagram(Info, Generic[_AT]): +class Datagram(DeferredPacket, Info, Generic[_AT]): """Data model for :term:`DTLS ` reassembled - handshake message. + handshake message.""" - :attr:`body` is reported eagerly, not through a - :class:`~pcapkit.foundation.reassembly.data.data.Deferred`: a message only - reaches :meth:`DTLS.submit ` - because it was fragmented, so there is no common unfragmented case to - keep cheap. - - """ + #: Listing ``packet`` here is what makes :attr:`packet` lazy -- see + #: :class:`~pcapkit.corekit.packet.DeferredPacket`. + __additional__ = ['packet'] + #: Placeholder class :attr:`packet` may hold, for + #: :class:`~pcapkit.corekit.packet.DeferredPacket` to resolve. + __deferred__ = Deferred #: How completely the message was reassembled, and why reassembly #: stopped; see :class:`~pcapkit.foundation.reassembly.data.data.Completion`. @@ -106,12 +108,17 @@ class Datagram(Info, Generic[_AT]): msg_type: 'Enum_HandshakeType' #: Total message length, as declared by its fragments. length: 'int' - #: Reassembled message body, undecoded; - #: :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` decodes it. + #: Reassembled message body, undecoded. body: 'bytes | tuple[bytes, ...]' + #: Decoded message, as :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` + #: returns it: decoded on first read, not at construction time, and + #: :data:`None` for an incomplete message. A + #: :class:`~pcapkit.foundation.reassembly.data.data.Deferred` may be passed in + #: its place, and reading this attribute then runs it and keeps the result. + packet: 'Optional[Data_DTLSHandshake]' if TYPE_CHECKING: - def __init__(self, completed: 'Completion', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', message_seq: 'int', msg_type: 'Enum_HandshakeType', length: 'int', body: 'bytes | tuple[bytes, ...]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, completed: 'Completion', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', message_seq: 'int', msg_type: 'Enum_HandshakeType', length: 'int', body: 'bytes | tuple[bytes, ...]', packet: 'Optional[Data_DTLSHandshake | Deferred[tuple[Enum_HandshakeType, int, int]]]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final diff --git a/pcapkit/foundation/reassembly/dtls.py b/pcapkit/foundation/reassembly/dtls.py index 74efe62d81..493f3731f0 100644 --- a/pcapkit/foundation/reassembly/dtls.py +++ b/pcapkit/foundation/reassembly/dtls.py @@ -16,13 +16,17 @@ """ from typing import TYPE_CHECKING, Generic -from pcapkit.foundation.reassembly.data.data import Completion +from pcapkit.foundation.analysis.dtls import _read_handshake +from pcapkit.foundation.reassembly.data.data import Completion, Deferred from pcapkit.foundation.reassembly.data.dtls import (_AT, Buffer, BufferID, Datagram, DatagramID, Fragment, HoleDescriptor, Packet) from pcapkit.foundation.reassembly.reassembly import ReassemblyBase +from pcapkit.protocols.application.dtls import DTLS as DTLS_Protocol if TYPE_CHECKING: - from typing import Optional + from typing import Optional, Type + + from pcapkit.const.tls.handshake_type import HandshakeType as Enum_HandshakeType __all__ = ['DTLS'] @@ -47,7 +51,9 @@ class DTLS(ReassemblyBase[Packet[_AT], Datagram[_AT], BufferID, Buffer], Generic :class:`~pcapkit.foundation.extraction.Extractor` feeds it through :func:`pcapkit.toolkit.pcap.dtls_reassembly`, which reads the 4-tuple off the frame, and :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` - decodes what it reassembles. + decodes what it reassembles; a complete message's + :attr:`~pcapkit.foundation.reassembly.data.dtls.Datagram.packet` holds + that decode, run on first read. Example: >>> from pcapkit.foundation.reassembly.dtls import DTLS @@ -59,17 +65,17 @@ class DTLS(ReassemblyBase[Packet[_AT], Datagram[_AT], BufferID, Buffer], Generic >>> result = dtls_reassembly.datagram """ + if TYPE_CHECKING: + protocol: 'Type[DTLS_Protocol]' ########################################################################## # Defaults. ########################################################################## - #: Protocol name of current reassembly object. Not set via - #: ``__protocol_type__``, which would import - #: :class:`~pcapkit.protocols.application.dtls.DTLS` at class-definition - #: time; :class:`~pcapkit.foundation.reassembly.reassembly.ReassemblyMeta` - #: looks :attr:`name` up in the protocol registry instead. + #: Protocol name of current reassembly object. __protocol_name__ = 'DTLS' + #: Protocol of current reassembly object. + __protocol_type__ = DTLS_Protocol # No ``__timeout__``: RFC 6347 section 4.2.3 and RFC 9147 section 5.5 give # DTLS handshake reassembly no timeout, so it inherits the base default, @@ -236,11 +242,17 @@ def _datagram_from_fragment(self, frag: 'Fragment', bufid: 'BufferID', message_s completion: how completely it was reassembled Returns: - The reassembled (or partially reassembled) message. + The reassembled (or partially reassembled) message. Its ``packet`` + is a deferred :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` + decode if the message is complete, and :data:`None` otherwise, as + that function returns. """ + packet = None # type: Optional[Deferred[tuple[Enum_HandshakeType, int, int]]] if completion: - payload = bytes(frag.raw) # type: 'bytes | tuple[bytes, ...]' + body = bytes(frag.raw) + payload = body # type: 'bytes | tuple[bytes, ...]' + packet = Deferred(_read_handshake, (frag.msg_type, frag.length, message_seq), body) elif self._flag_s: payload = self._received_runs(frag) else: @@ -260,6 +272,7 @@ def _datagram_from_fragment(self, frag: 'Fragment', bufid: 'BufferID', message_s msg_type=frag.msg_type, length=frag.length, body=payload, + packet=packet, ) @staticmethod diff --git a/tests/corekit/test_deferred_packet_1516_unit.py b/tests/corekit/test_deferred_packet_1516_unit.py index 1b475423eb..70ae28745d 100644 --- a/tests/corekit/test_deferred_packet_1516_unit.py +++ b/tests/corekit/test_deferred_packet_1516_unit.py @@ -10,7 +10,7 @@ :class:`DeferredPacketTests` pins the shared class against a placeholder of its own, so it holds whatever either subsystem later does with the mixin. -:class:`WiringTests` pins the three real models. +:class:`WiringTests` pins the real models, DTLS reassembly's among them (#2531). Each test imports what it needs itself, rather than the module importing it at the top. On a tree without :mod:`pcapkit.corekit.packet`, every test then fails @@ -27,10 +27,11 @@ #: The module under test, read as source by the import-layering check. MODULE = pathlib.Path(__file__).resolve().parents[2] / 'pcapkit' / 'corekit' / 'packet.py' -#: The three models, as ``(module, class, module holding its placeholder)``. +#: The models, as ``(module, class, module holding its placeholder)``. MODELS = ( ('pcapkit.foundation.reassembly.data.ip', 'Datagram', 'pcapkit.foundation.reassembly.data.data'), ('pcapkit.foundation.reassembly.data.tcp', 'Datagram', 'pcapkit.foundation.reassembly.data.data'), + ('pcapkit.foundation.reassembly.data.dtls', 'Datagram', 'pcapkit.foundation.reassembly.data.data'), ('pcapkit.foundation.traceflow.data.tcp', 'Index', 'pcapkit.foundation.traceflow.data.data'), ) @@ -198,7 +199,7 @@ def test_the_module_imports_nothing_from_foundation_or_protocols(self) -> 'None' class WiringTests(unittest.TestCase): - """The three real models share the one class, each with its own placeholder.""" + """The real models share the one class, each with its own placeholder.""" def test_every_model_shares_the_one_class_mixin_first(self) -> 'None': from pcapkit.corekit.packet import DeferredPacket @@ -231,6 +232,7 @@ def test_each_model_resolves_its_own_placeholder_and_not_the_other(self) -> 'Non from pcapkit.corekit.packet import DeferredPacket from pcapkit.foundation.reassembly.data.data import Completion from pcapkit.foundation.reassembly.data.data import Deferred as ReassemblyDeferred + from pcapkit.foundation.reassembly.data.dtls import Datagram as DTLS_Datagram from pcapkit.foundation.reassembly.data.ip import Datagram as IP_Datagram from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram from pcapkit.foundation.traceflow.data.data import Deferred as TraceFlowDeferred @@ -249,11 +251,16 @@ def datagram(cls: 'type', packet: 'object') -> 'DeferredPacket': def index(packet: 'object') -> 'DeferredPacket': return Index(fpout=None, index=(1,), label='label', forward=(1,), reverse=(), packet=packet) + def message(packet: 'object') -> 'DeferredPacket': + return DTLS_Datagram(completed=Completion.COMPLETE, id=None, index=(1,), message_seq=0, + msg_type=None, length=7, body=b'payload', packet=packet) + cases = ( ('ip', lambda packet: datagram(IP_Datagram, packet), reassembly, traceflow, ('parsed', 'proto', b'payload')), ('tcp', lambda packet: datagram(TCP_Datagram, packet), reassembly, traceflow, ('parsed', 'proto', b'payload')), + ('dtls', message, reassembly, traceflow, ('parsed', 'proto', b'payload')), ('trace', index, traceflow, reassembly, ('reassembled',)), ) for (label, build, own, other, expected) in cases: diff --git a/tests/foundation/reassembly/test_deferred_proto_type_2417_unit.py b/tests/foundation/reassembly/test_deferred_proto_type_2417_unit.py index 91c1882109..79d6130890 100644 --- a/tests/foundation/reassembly/test_deferred_proto_type_2417_unit.py +++ b/tests/foundation/reassembly/test_deferred_proto_type_2417_unit.py @@ -229,9 +229,10 @@ def _declared_proto_types(cls): """What that type variable is constrained to, each one evaluated. A constraint is written as a string literal, since - :class:`~pcapkit.const.reg.transtype.TransType` and - :class:`~pcapkit.const.reg.apptype.tcp.TCP` are both imported - under ``TYPE_CHECKING`` and nothing at runtime has either in scope; the + :class:`~pcapkit.const.reg.transtype.TransType`, + :class:`~pcapkit.const.reg.apptype.tcp.TCP` and + :class:`~pcapkit.const.tls.handshake_type.HandshakeType` are imported + under ``TYPE_CHECKING`` and nothing at runtime has any in scope; the namespace is rebuilt here rather than asking :func:`typing.get_type_hints`, which raises :exc:`NameError` on them. :class:`~typing.TypeVar` keeps such a constraint as the string itself @@ -244,10 +245,12 @@ def _declared_proto_types(cls): """ from pcapkit.const.reg.apptype import TCP as AppType_TCP from pcapkit.const.reg.transtype import TransType + from pcapkit.const.tls.handshake_type import HandshakeType return tuple( eval(getattr(constraint, '__forward_arg__', constraint), # pylint: disable=eval-used - {'TransType': TransType, 'AppType_TCP': AppType_TCP}) + {'TransType': TransType, 'AppType_TCP': AppType_TCP, + 'HandshakeType': HandshakeType}) for constraint in cls._proto_type_variable().__constraints__ ) @@ -267,6 +270,27 @@ def _tcp_proto(self, bufid=TCP_BUFID): """The ``proto`` ``TCP.submit`` hands to :class:`Deferred`.""" return tcp_proto(bufid) + def _dtls_proto(self): + """The ``proto`` DTLS reassembly hands :class:`Deferred` for a complete + message (#2531): its ``(msg_type, length, message_seq)``.""" + from pcapkit.const.tls.handshake_type import HandshakeType + from pcapkit.foundation.reassembly.data.dtls import Packet + from pcapkit.foundation.reassembly.dtls import DTLS + + reasm = DTLS() + reasm(Packet((ip_address('192.0.2.1'), 1234, ip_address('198.51.100.2'), 5684, 0), 1, + HandshakeType.certificate, 5, 7, 0, 5, b'hello', 1000.0)) + datagram, = reasm.datagram + return held(datagram).proto + + def test_dtls_passes_the_handshake_header(self) -> None: + """The body decodes by ``msg_type``; ``length`` and ``message_seq`` frame it.""" + from pcapkit.const.tls.handshake_type import HandshakeType + + msg_type, length, message_seq = self._dtls_proto() + self.assertIs(msg_type, HandshakeType.certificate) + self.assertEqual((length, message_seq), (5, 7)) + def test_ip_passes_the_payload_protocol_type(self) -> None: """IP reassembly reads it from the datagram's **Protocol** field.""" from pcapkit.const.reg.transtype import TransType @@ -307,14 +331,15 @@ def test_the_analyser_takes_the_same_type_variable_as_proto(self) -> None: def test_declared_type_admits_what_each_reassembler_passes(self) -> None: """The regression: the port pair was outside the declared type. - Each value has to land on *exactly one* constraint, and the two on - different ones -- otherwise the type parameter is not distinguishing the - two reassemblers and a union would have done. + Each value has to land on *exactly one* constraint, and each on a + different one -- otherwise the type parameter is not distinguishing the + reassemblers and a union would have done. """ declared = self._declared_proto_types() matched = {} - for name, value in (('ip', self._ip_proto()), ('tcp', self._tcp_proto())): + values = (('ip', self._ip_proto()), ('tcp', self._tcp_proto()), ('dtls', self._dtls_proto())) + for name, value in values: with self.subTest(reassembler=name): admitting = [each for each in declared if admits(each, value)] self.assertEqual(len(admitting), 1, @@ -322,12 +347,11 @@ def test_declared_type_admits_what_each_reassembler_passes(self) -> None: f'one of {declared!r}') matched[name] = admitting[0] # NOTE: a failing ``subTest`` does not stop the loop, so one of these may - # be absent; reading it anyway would raise :exc:`KeyError` over the real - # failure the subtest has already reported. - if len(matched) == 2: - self.assertNotEqual(matched['ip'], matched['tcp'], - f'both reassemblers resolved to {matched["ip"]!r}, so the type ' - 'parameter distinguishes nothing a union would not') + # be absent; comparing anyway would report a collision that is not there. + if len(matched) == len(values): + self.assertEqual(len(set(matched.values())), len(values), + f'reassemblers share a constraint in {matched!r}, so the type ' + 'parameter distinguishes nothing a union would not') def test_the_held_analyser_is_not_called_to_learn_this(self) -> None: """``packet`` is still deferred after ``proto`` has been read.""" diff --git a/tests/foundation/reassembly/test_dtls_reassembly_parity_2531_unit.py b/tests/foundation/reassembly/test_dtls_reassembly_parity_2531_unit.py new file mode 100644 index 0000000000..02b583d9fb --- /dev/null +++ b/tests/foundation/reassembly/test_dtls_reassembly_parity_2531_unit.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +"""DTLS reassembly takes the IP/TCP shape (#2531). + +``reassembly-integration.rst`` listed the ways DTLS reassembly still differed from +IP and TCP once #2528 routed it through the toolkit and the analysis: + +* a reassembled message carried no ``packet``, where an IP or TCP datagram's is a + :class:`~pcapkit.foundation.reassembly.data.data.Deferred` parse run on first + read -- for DTLS, the decoding + :func:`~pcapkit.foundation.analysis.dtls.dtls_handshake` does; +* :class:`~pcapkit.foundation.reassembly.dtls.DTLS` set no ``__protocol_type__``, + so its protocol was looked up in the protocol registry by name. + +Each case runs the default engine over a capture built in memory, as PCAP and as +PCAP-NG, and reads no sample capture. + +""" + +from __future__ import annotations + +import pickle +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.foundation.reassembly.test_dtls_handshake_reassembly_2440_unit import ( + _client_hello_body, _dtls_handshake, _dtls_record, _extract, _frame, _pcap, _pcapng, _split) + +if TYPE_CHECKING: + from typing import Any + +#: Test tier (#1958): pickling, whose protocols differ by interpreter. +__tier__ = 'ALL' + +#: A ``ClientHello`` body, the message the captures below fragment. +BODY = _client_hello_body(cookie=b'\xab\xcd\xef\x01') +#: Its message sequence number. +MESSAGE_SEQ = 4 +#: The containers each case runs over. +CONTAINERS = (('pcap', _pcap), ('pcapng', _pcapng)) + + +def _records(frags: 'list[tuple[int, int]]') -> 'list[bytes]': + """One ``handshake`` record per ``(offset, length)`` fragment of :data:`BODY`.""" + return [_dtls_record(22, _dtls_handshake(1, BODY[fo:fo + length], message_seq=MESSAGE_SEQ, + fragment_offset=fo, fragment_length=length, + total_length=len(BODY)), seq=num) + for num, (fo, length) in enumerate(frags, 1)] + + +#: The whole message, in two fragments. +COMPLETE = _records(_split(BODY, (len(BODY) // 2,))) +#: The first and last thirds only; the middle one is lost. +INCOMPLETE = [record for num, record in + enumerate(_records(_split(BODY, (len(BODY) // 3, 2 * len(BODY) // 3)))) if num != 1] + + +def _held(datagram: 'Any') -> 'Any': + """``datagram``'s ``packet`` as stored, without resolving it.""" + return datagram.__dict__[datagram.__map__.get('packet', 'packet')] + + +class DeferredPacketTests(unittest.TestCase): + """A reassembled message's ``packet`` is ``dtls_handshake``'s result, deferred.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def _datagram(self, write: 'Any', records: 'list[bytes]', **kwargs: 'Any') -> 'Any': + extractor = _extract(self, write([_frame(record) for record in records]), + reassembly=True, dtls=True, **kwargs) + datagram, = extractor.reassembly.dtls + return datagram + + def test_a_complete_message_holds_a_deferred_decode(self) -> None: + from pcapkit.const.tls.handshake_type import HandshakeType + from pcapkit.foundation.reassembly.data.data import Deferred + + for container, write in CONTAINERS: + with self.subTest(container=container): + held = _held(self._datagram(write, COMPLETE)) + self.assertIsInstance(held, Deferred) + self.assertEqual(held.proto, (HandshakeType.client_hello, len(BODY), MESSAGE_SEQ)) + self.assertEqual(held.payload, BODY) + + def test_reading_packet_decodes_as_dtls_handshake_does(self) -> None: + from pcapkit.foundation.analysis import dtls_handshake + from pcapkit.protocols.data.application.tls import ClientHello + + for container, write in CONTAINERS: + with self.subTest(container=container): + datagram = self._datagram(write, COMPLETE) + packet = datagram.packet + self.assertEqual(packet, dtls_handshake(datagram)) + self.assertIsInstance(packet.body, ClientHello) + self.assertEqual(packet.body.cookie, b'\xab\xcd\xef\x01') + self.assertEqual((packet.message_seq, packet.fragment_offset, packet.fragment_length, + packet.is_fragmented), (MESSAGE_SEQ, 0, len(BODY), False)) + self.assertIs(datagram.packet, packet) + self.assertEqual(datagram.to_dict()['packet'], packet.to_dict()) + + def test_the_decode_runs_on_first_read_and_only_once(self) -> None: + from pcapkit.protocols.application.dtls import DTLS + + for container, write in CONTAINERS: + with self.subTest(container=container): + with mock.patch.object(DTLS, 'read_handshake', wraps=DTLS.read_handshake) as decode: + datagram = self._datagram(write, COMPLETE) + self.assertIn('packet', datagram) + self.assertEqual(decode.call_count, 0) + datagram.packet # pylint: disable=pointless-statement + datagram['packet'] # pylint: disable=pointless-statement + repr(datagram) + self.assertEqual(decode.call_count, 1) + + def test_an_incomplete_message_has_no_packet(self) -> None: + from pcapkit.foundation.analysis import dtls_handshake + + for container, write in CONTAINERS: + for strict in (True, False): + with self.subTest(container=container, strict=strict): + datagram = self._datagram(write, INCOMPLETE, reasm_strict=strict) + self.assertEqual(datagram.completed.name, 'PARTIAL') + self.assertIsInstance(datagram.body, tuple if strict else bytes) + self.assertIsNone(_held(datagram)) + self.assertIsNone(datagram.packet) + self.assertIsNone(dtls_handshake(datagram)) + + def test_a_deferred_datagram_pickles_and_still_decodes(self) -> None: + from pcapkit.foundation.analysis import dtls_handshake + from pcapkit.foundation.reassembly.data.data import Deferred + + for container, write in CONTAINERS: + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(container=container, protocol=protocol): + datagram = self._datagram(write, COMPLETE) + restored = pickle.loads(pickle.dumps(datagram, protocol=protocol)) + self.assertIsInstance(_held(restored), Deferred) + self.assertEqual(restored.packet, dtls_handshake(datagram)) + + +class ProtocolTypeTests(unittest.TestCase): + """The engine names its protocol itself, as the IP and TCP engines do.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_the_engine_declares_the_dtls_protocol(self) -> None: + from pcapkit.protocols.application.dtls import DTLS + + for container, write in CONTAINERS: + with self.subTest(container=container): + extractor = _extract(self, write([_frame(record) for record in COMPLETE]), + reassembly=True, dtls=True) + engine = type(extractor._reasm.dtls) + self.assertIs(vars(engine).get('__protocol_type__'), DTLS) + self.assertIs(engine.protocol, DTLS) + + def test_the_protocol_survives_a_registry_without_it(self) -> None: + """With no ``'DTLS'`` registered, a name lookup would answer ``Raw``.""" + import pcapkit.protocols + from pcapkit.protocols.application.dtls import DTLS + + for container, write in CONTAINERS: + with self.subTest(container=container): + extractor = _extract(self, write([_frame(record) for record in COMPLETE]), + reassembly=True, ip=True, tcp=True, dtls=True) + with mock.patch.dict(pcapkit.protocols.__proto__): + for name in ('IPV4', 'TCP', 'DTLS'): + del pcapkit.protocols.__proto__[name] + for name in ('ipv4', 'tcp', 'dtls'): + engine = getattr(extractor._reasm, name) + self.assertIs(engine.protocol, type(engine).__protocol_type__, name) + self.assertIs(extractor._reasm.dtls.protocol, DTLS) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/reassembly/test_dtls_toolkit_analysis_2496_unit.py b/tests/foundation/reassembly/test_dtls_toolkit_analysis_2496_unit.py index 54f932186c..7552db3a77 100644 --- a/tests/foundation/reassembly/test_dtls_toolkit_analysis_2496_unit.py +++ b/tests/foundation/reassembly/test_dtls_toolkit_analysis_2496_unit.py @@ -148,7 +148,7 @@ def _datagram(body: 'bytes | tuple[bytes, ...]', *, msg_type: int = 1, return Datagram(completed=Completion[completed], id=DatagramID(*FLOW, 0), index=(1, 2), message_seq=4, msg_type=HandshakeType.get(msg_type), - length=len(body) if length is None else length, body=body) + length=len(body) if length is None else length, body=body, packet=None) def test_a_hello_decodes_as_the_unfragmented_message_does(self) -> None: from pcapkit.foundation.analysis import dtls_handshake