diff --git a/pcapkit/corekit/fields/collections.py b/pcapkit/corekit/fields/collections.py index 17ba1dd80c..8b34581fcc 100644 --- a/pcapkit/corekit/fields/collections.py +++ b/pcapkit/corekit/fields/collections.py @@ -213,15 +213,18 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> 'byte consumes nothing from ``buffer``, or if schema items short of the field's length cannot be zero-filled. - When ``buffer`` falls short of the field's length, schema items are read - from its octets zero-filled to that length, as + When ``buffer`` falls short of the field's length, the schema item it + ends inside is read as though its missing octets were zeros, as :meth:`FieldBase.unpack ` - reads any other field, so the items cover the length declared. + reads any other field, and the list ends with that item: none is read + from the zeros past it, which hold no octet of the data. .. versionchanged:: 1.6.0 - Schema items short of the field's length are zero-filled to it, where - an item past the data was refused as consuming nothing (:issue:`2686`). + A list of schema items the data ends inside ends with the data, the + item it ends inside read with its missing octets as zeros, where an + item past the data was refused as consuming nothing (:issue:`2686`, + :issue:`2756`). """ length = self._length @@ -267,16 +270,20 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> 'byte file.seek(start, io.SEEK_SET) if present < length: with _zero_filled_items(self.name, length, file.read()) as filled: - return self._unpack_items(io.BytesIO(filled), packet, is_schema) + return self._unpack_items(filled, packet, is_schema, present) return self._unpack_items(file, packet, is_schema) - def _unpack_items(self, file: 'IO[bytes]', packet: 'dict[str, Any]', is_schema: 'bool') -> 'list[_TL]': + def _unpack_items(self, file: 'IO[bytes]', packet: 'dict[str, Any]', is_schema: 'bool', + present: 'int' = -1) -> 'list[_TL]': """Unpack the field's items from ``file``. Args: file: Stream the items are read from. packet: Packet data. is_schema: Whether the items are schemas. + present: Octets of ``file`` the data holds, past which it holds the + zeros :meth:`unpack` filled it with; negative where it holds + no zeros. Returns: Unpacked items. @@ -293,8 +300,9 @@ def _unpack_items(self, file: 'IO[bytes]', packet: 'dict[str, Any]', is_schema: # which is what it read, but the schema branch sizes it by ``len(data)``, # which is only what the schema *recorded*. A schema that reads nothing # records nothing, so ``length -= len(data)`` makes no progress and the - # loop spins forever. The zero-fill in ``unpack`` gives every item octets, - # but an item schema may still consume none of them. Remembering where the + # loop spins forever. The zero-fill in ``unpack`` gives the item the data + # ends inside octets to read, and every item before it reads octets of the + # data, but an item schema may still consume none of them. Remembering where the # previous item ended is what bounds the iteration count, since it does # not depend on what the schema reports. Cf. #431, which is the same # defect in the ``OptionField`` subclass. @@ -308,6 +316,10 @@ def _unpack_items(self, file: 'IO[bytes]', packet: 'dict[str, Any]', is_schema: temp = [] # type: list[_TL] while length > 0: + # NOTE: An item starting past the octets present would be made of the zero-fill alone, + # one the capture never held, so the list ends with the data (#2756). + if 0 <= present <= offset - start: + break field = item_type(packet) if is_schema: diff --git a/pcapkit/corekit/fields/field.py b/pcapkit/corekit/fields/field.py index 7dcb03815f..9a5e451fe5 100644 --- a/pcapkit/corekit/fields/field.py +++ b/pcapkit/corekit/fields/field.py @@ -23,6 +23,7 @@ import abc import contextlib import contextvars +import io import re import struct from typing import TYPE_CHECKING, Generic, TypeVar, cast @@ -120,14 +121,15 @@ def _zero_pad_budget() -> 'Iterator[list[int]]': @contextlib.contextmanager -def _zero_filled_items(name: 'str', length: 'int', data: 'bytes') -> 'Iterator[bytes]': +def _zero_filled_items(name: 'str', length: 'int', data: 'bytes') -> 'Iterator[IO[bytes]]': """Zero-fill the octets of a list of schema items to its declared width. :meth:`FieldBase.unpack` reads a field the data ends inside as though its missing octets were zeros, and :meth:`ListField.unpack - ` gives a list of - schema items the same reading here, so its items cover the width it - declares (:issue:`2686`). The items are read inside the block. + ` gives the schema + item the data ends inside the same reading here (:issue:`2686`). The items + are read inside the block, and none from the zeros past that item, which + hold no octet of the data (:issue:`2756`). Args: name: Field name. @@ -135,7 +137,8 @@ def _zero_filled_items(name: 'str', length: 'int', data: 'bytes') -> 'Iterator[b data: Octets actually present, fewer than ``length``. Yields: - ``data``, zero-filled to ``length``. + A stream of ``data``, zero-filled to ``length``. The zeros left unread + past where the stream ends up are not counted as octets read. Raises: FieldValueError: If ``length`` is past @@ -157,8 +160,8 @@ def _zero_filled_items(name: 'str', length: 'int', data: 'bytes') -> 'Iterator[b ledger = [0, 0] _zero_pad_ledger.set(ledger) - # NOTE: The fill is made-up schema objects, not zeros, so it gets no free band or one-off - # allowance: a 15-octet SETTINGS frame declaring 65,532 octets made 10,922 pairs in 7.5 MB. + # NOTE: The fill makes up the rest of the item the data ends inside, a schema object, not + # zeros, so it gets no free band or one-off allowance (#2686); past that item it makes none. padding = length - len(data) shortfall, ratio = config.MAX_ZERO_PAD_SHORTFALL, config.ZERO_PAD_BUDGET_RATIO if shortfall != -1: @@ -174,14 +177,17 @@ def _zero_filled_items(name: 'str', length: 'int', data: 'bytes') -> 'Iterator[b ledger[1] += padding # NOTE: The items credit the zeros they read as octets supplied, so they are taken off in - # advance; items that fail credit an unknown share, so the octets read go back as they were. + # advance and those left unread given back (#2756); items that fail credit an unknown share, + # so the octets read go back as they were. read = ledger[0] ledger[0] -= padding + filled = io.BytesIO(data + bytes(padding)) try: - yield data + bytes(padding) + yield filled except BaseException: ledger[0] = read raise + ledger[0] += length - max(filled.tell(), len(data)) #: Pattern[str]: Matches a leading, optionally byte-order-prefixed negative diff --git a/pcapkit/protocols/application/httpv2/protocol.py b/pcapkit/protocols/application/httpv2/protocol.py index 42a5fbc91f..cf60e88407 100644 --- a/pcapkit/protocols/application/httpv2/protocol.py +++ b/pcapkit/protocols/application/httpv2/protocol.py @@ -207,9 +207,10 @@ class HTTPv2(HTTPBase[Data_HTTP, Schema_HTTP], class, where it was kept as :class:`~pcapkit.protocols.misc.raw.protocol.Raw` (:issue:`2536`). * A ``SETTINGS`` frame the capture cut inside its settings parses, with the - settings past the data read as zeros, as a cut ``DATA`` frame's data is, - where it was :class:`~pcapkit.protocols.misc.raw.protocol.Raw` (:issue:`2659`, - :issue:`2686`). + setting the cut lands inside read as zeros past it, as a cut ``DATA`` + frame's data is, and no setting past the data, where it was + :class:`~pcapkit.protocols.misc.raw.protocol.Raw` (:issue:`2659`, + :issue:`2686`, :issue:`2756`). * Each enumeration field of :meth:`make` and of the frame makers takes a member or an :class:`int` code, resolved with the enumeration's ``get``; a name, a :obj:`bool` for a field wider than one bit, or a @@ -1437,6 +1438,15 @@ def _make_http_settings(self, frame: 'Optional[Data_SettingsFrame]' = None, *, frame with no parameters, which :rfc:`9113#section-6.5` permits, rather than being rejected (:issue:`2274`). + .. versionchanged:: 1.6.0 + + A ``frame`` the capture cut inside its settings, as its + ``__short_read__`` records, packs the settings its parse read from the + octets its ``packet`` holds with zeros after them to the Length it + declares, since that parse reads no setting past the data, so its + rebuild declares that Length too; with its settings edited, the Length + is computed as for a whole frame (:issue:`2756`). + """ if frame is not None: ack = frame.flags.ACK @@ -1478,6 +1488,16 @@ def _make_http_settings(self, frame: 'Optional[Data_SettingsFrame]' = None, *, raise ProtocolError(f'HTTP/2: [Type {Enum_Frame.SETTINGS}] invalid settings: ' f'{settings!r}') + # NOTE: An unedited cut frame packs the octets its ``packet`` captured, zero to the pair, and + # keeps its Length in the zeros past them (#2756); a dict rebuilds as the base data model. + record = frame.get('__short_read__') if isinstance(frame, Data_HTTP) else None + if frame is not None and record is not None and isinstance(settings_val, list): + packed = b''.join(pair.pack() for pair in settings_val) + # A cut in the frame header leaves no setting captured. + read = cast('bytes', frame.get('packet', b''))[:record[1]] if record[0] == 'frame' else b'' + if packed == read + bytes(-len(read) % 6) and len(packed) < frame.length: + settings_val = packed + bytes(frame.length - len(packed)) + return Schema_SettingsFrame( settings=settings_val, ), flags diff --git a/tests/corekit/test_fields_list_zero_fill_2686_unit.py b/tests/corekit/test_fields_list_zero_fill_2686_unit.py index f717662dc6..e30ffb457e 100644 --- a/tests/corekit/test_fields_list_zero_fill_2686_unit.py +++ b/tests/corekit/test_fields_list_zero_fill_2686_unit.py @@ -5,12 +5,13 @@ field the data ends inside as though its missing octets were zeros. :meth:`ListField.unpack ` refused the same reading for schema items: an item past the data consumed nothing, -and the loop guard raised. Now the items read the octets present zero-filled to the -field's length, so a cut list covers the length it declares and rebuilds from it. +and the loop guard raised. The item the data ends inside reads instead the octets +present zero-filled to the field's length, and the list ends with it: no item is read +past it from the zeros alone, which hold no octet of the data (#2756). -Every zero-filled item is a schema object rather than a zero octet, so the fill is +The item the fill completes is a schema object rather than a zero octet, so the fill is charged to the running budget with no free band and no one-off allowance, and the -zeros the items then read earn no allowance of their own. +zeros the items read earn no allowance of their own, nor do those they leave unread. Every case builds its own octets. @@ -72,7 +73,7 @@ def test_a_short_buffer_and_stream_are_zero_filled_alike(self) -> None: for name, buffer in (('bytes', present), ('stream', stream)): with self.subTest(buffer=name), self.budget(), time_limit(): parsed = self.field(18).unpack(buffer, {}) - self.assertEqual(self.pairs(parsed), [(1, 4096), (3, 0), (0, 0)]) + self.assertEqual(self.pairs(parsed), [(1, 4096), (3, 0)]) self.assertEqual(stream.tell(), 2 + len(present)) def test_a_whole_list_is_read_as_it_was(self) -> None: @@ -86,14 +87,14 @@ def test_a_whole_list_is_read_as_it_was(self) -> None: self.assertEqual(stream.tell(), 12) def test_a_cut_schema_packs_back_as_captured(self) -> None: - """The short read is recorded on the list, which packs at its full length and is - cut back to the octets captured, from the schema and from its dict.""" + """The short read is recorded on the list, which holds the pairs the octets + captured begin and is cut back to those octets, from the schema and from its dict.""" whole = bytes.fromhex('0003' '000100001000' '000300000064' '000400010000') for keep in range(2, len(whole)): cut = whole[:keep] with self.subTest(keep=keep): schema = self.Pairs.unpack(cut, len(whole)) - self.assertEqual(len(schema.pairs), 3) + self.assertEqual(len(schema.pairs), -(-(keep - 2) // 6)) self.assertEqual(schema.__dict__['__short_read__'], ('pairs', keep - 2)) self.assertEqual(schema.pack(), cut) self.assertEqual(self.Pairs.from_dict(schema.to_dict()).pack(), cut) @@ -124,7 +125,8 @@ def test_the_zeros_earn_no_allowance(self) -> None: def test_a_failed_fill_gives_back_the_octets_read(self) -> None: """Items that fail inside the fill leave the octets read as they were, so one bad list does not refuse the later cuts of its context; the fill stays charged. - Only a failure restores them: a later fill that succeeds is counted as usual.""" + Only a failure restores them: a later fill that succeeds is counted as usual, + the zeros it leaves unread past its items given back (#2756).""" from pcapkit.corekit.fields.numbers import UInt16Field from pcapkit.protocols.schema import Schema, schema_final from pcapkit.utilities.exceptions import ProtocolError @@ -145,12 +147,12 @@ def post_process(self, packet: 'dict') -> 'Schema': self.field(12).unpack(bytes(12), {}) self.assertEqual(ledger, [12, 0]) with self.assertRaisesRegex(ProtocolError, 'a zero item'): - field.unpack(b'\x00\x01', {}) - self.assertEqual(ledger, [12, 10]) + field.unpack(b'\x00', {}) + self.assertEqual(ledger, [12, 11]) # a fill that succeeds keeps the octets its items read, as before the failure self.field(12).unpack(bytes(2), {}) - self.assertEqual(ledger, [14, 20]) + self.assertEqual(ledger, [14, 21]) def test_a_fresh_context_gets_a_ledger_of_its_own(self) -> None: """A fill that is the first read of a context starts that context's ledger.""" @@ -160,7 +162,7 @@ def fill() -> 'tuple[int, list[int]]': parsed = self.field(12).unpack(bytes(6), {}) return len(parsed), self.ledger.get() - self.assertEqual(contextvars.Context().run(fill), (2, [6, 6])) + self.assertEqual(contextvars.Context().run(fill), (1, [6, 6])) def test_a_length_past_the_ceiling_is_refused(self) -> None: ceiling = self.config.MAX_ZERO_PAD_LENGTH @@ -171,7 +173,7 @@ def test_a_length_past_the_ceiling_is_refused(self) -> None: def test_a_lifted_budget_fills_any_shortfall_within_the_ceiling(self) -> None: for name in ('ZERO_PAD_BUDGET_RATIO', 'MAX_ZERO_PAD_SHORTFALL'): with self.subTest(name), mock.patch.object(self.config, name, -1), self.budget(), time_limit(): - self.assertEqual(len(self.field(6000).unpack(b'', {})), 1000) + self.assertEqual(len(self.field(6000).unpack(bytes(1), {})), 1) if __name__ == '__main__': diff --git a/tests/protocols/application/test_httpv2_settings_cut_2659_unit.py b/tests/protocols/application/test_httpv2_settings_cut_2659_unit.py index 006eb23e1a..0c55e0c87b 100644 --- a/tests/protocols/application/test_httpv2_settings_cut_2659_unit.py +++ b/tests/protocols/application/test_httpv2_settings_cut_2659_unit.py @@ -1,13 +1,15 @@ # -*- coding: utf-8 -*- -"""An HTTP/2 ``SETTINGS`` frame cut inside its settings parses and rebuilds (#2659, #2686). +"""An HTTP/2 ``SETTINGS`` frame cut inside its settings parses and rebuilds (#2659, #2686, #2756). A capture cut short hands the frame fewer octets than its Length, which the IPv4 total length still backs. ``SettingsFrame.settings`` is sized by that Length, so past the octets present a ``SettingPair`` consumed nothing and ``ListField.unpack`` raised -``FieldValueError``, leaving the segment ``Raw``. The list now reads its settings -zero-filled to the Length, as a cut ``DATA`` frame reads its data, so ``make`` keeps -computing the Length from the frame it builds, and an edited whole frame rebuilds with -the Length of its edited payload. +``FieldValueError``, leaving the segment ``Raw``. The list reads the setting the data +ends inside zero-filled, as a cut ``DATA`` frame reads its data, and makes up none past +the data from the zeros (#2756). The rebuild of a cut frame packs zeros after its +settings to the Length it declares, while ``make`` computes the Length of a whole frame +from the frame it builds, so an edited whole frame rebuilds with the Length of its +edited payload. Every case builds its own octets. @@ -41,8 +43,8 @@ def _datagram(app: bytes) -> bytes: def _expected(present: bytes) -> 'list[tuple[int, int]]': - """The pairs ``present`` holds, zero-filled to the frame's Length as a short read is.""" - padded = present + bytes(len(PAIRS) - len(present)) + """The pairs ``present`` begins, the last zero-filled as a short read is.""" + padded = present + bytes(-len(present) % 6) return [(int.from_bytes(padded[i:i + 2], 'big'), int.from_bytes(padded[i + 2:i + 6], 'big')) for i in range(0, len(padded), 6)] @@ -95,17 +97,18 @@ def test_cut_through_httpv2_at_every_offset(self) -> None: self.assert_cut(HTTPv2(SETTINGS[:9 + keep], len(SETTINGS)), keep) self.assertEqual(caught, []) - def test_the_settings_past_the_data_read_as_zeros(self) -> None: - """The setting the data ends inside keeps its identifier, and every setting - past the data reads as identifier and value zero, as a cut ``DATA`` frame - reads its data as zeros (#2686).""" + def test_no_setting_is_made_up_past_the_data(self) -> None: + """The setting the data ends inside keeps its identifier and reads its value + as zeros, as a cut ``DATA`` frame reads its data (#2686), and none past the + data is read from the zeros, as identifier and value zero (#2756).""" from pcapkit.const.http.setting import Setting from pcapkit.protocols.application.httpv2.protocol import HTTPv2 http = HTTPv2(SETTINGS[:9 + 9], len(SETTINGS)) self.assertEqual(list(http.info.settings.items(multi=True)), - [(Setting.HEADER_TABLE_SIZE, 4096), (Setting.MAX_CONCURRENT_STREAMS, 0), - (Setting.get(0), 0)]) + [(Setting.HEADER_TABLE_SIZE, 4096), (Setting.MAX_CONCURRENT_STREAMS, 0)]) + self.assertEqual(list(HTTPv2(SETTINGS[:9 + 12], len(SETTINGS)).info.settings.items(multi=True)), + [(Setting.HEADER_TABLE_SIZE, 4096), (Setting.MAX_CONCURRENT_STREAMS, 100)]) data = HTTPv2(_frame(0x0, 0x1, 1, b'hello')[:9 + 2], 9 + 5) self.assertEqual(data.info.data, b'he\x00\x00\x00') @@ -150,8 +153,8 @@ def test_the_next_frame_is_not_swallowed(self) -> None: self.assertIsNone(rest.info.error) def test_a_length_no_capture_backs_goes_raw(self) -> None: - """A Length the octets read cannot back is refused, as zero-filling it would - make a setting per six octets of it, and the segment stays ``Raw`` (#2686).""" + """A Length the octets read cannot back is refused, as zero-filling it takes the + parse past its padding budget, and the segment stays ``Raw`` (#2686).""" from pcapkit.corekit.fields.field import _zero_pad_budget from pcapkit.protocols.application.httpv2.protocol import HTTPv2 from pcapkit.protocols.internet.ipv4.protocol import IPv4 @@ -170,7 +173,7 @@ def test_a_length_no_capture_backs_goes_raw(self) -> None: class SettingsListTests(unittest.TestCase): - """The settings list reads a buffer and a stream alike, zero-filled to its length.""" + """The settings list reads a buffer and a stream alike, ending with the data.""" def setUp(self) -> None: reimport_once_per_class(self) @@ -187,7 +190,7 @@ def test_reads_bytes_and_a_live_stream_alike(self) -> None: warnings.simplefilter('always') parsed = field.unpack(buffer, {}) self.assertEqual(caught, []) - self.assertEqual([(int(pair.id), pair.value) for pair in parsed], [(1, 4096), (3, 0), (0, 0)]) + self.assertEqual([(int(pair.id), pair.value) for pair in parsed], [(1, 4096), (3, 0)]) self.assertEqual(field.length, len(PAIRS)) self.assertEqual(stream.tell(), 2 + 9) @@ -265,6 +268,40 @@ def test_the_length_is_computed_when_not_given(self) -> None: self.assertEqual(http.info.length, 6) self.assertNotIn('__short_read__', http.info) + def test_an_edited_cut_frame_computes_its_length(self) -> None: + """A cut frame whose settings are edited, so no longer the pairs its capture began with + zeros past the cut, rebuilds with the Length of the edited settings, as an edited whole + frame does, not the Length the cut frame declares (#2756).""" + from pcapkit.corekit.multidict import OrderedMultiDict + from pcapkit.protocols.application.httpv2.protocol import HTTPv2 + from pcapkit.protocols.internet.ipv4.protocol import IPv4 + + packet = _datagram(SETTINGS) + head = len(packet) - len(SETTINGS) + whole = HTTPv2(SETTINGS, len(SETTINGS)).info + edits = ( + ('one pair', lambda pairs: pairs[:1]), + ('zero pairs', lambda pairs: []), + ('a value past the cut', lambda pairs: pairs[:-1] + [(pairs[-1][0], 0x01010101)]), + ) + # Past the first pair, so the pairs begun are more than one. + for keep in range(7, len(PAIRS)): + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + cut = IPv4(packet[:head + 9 + keep], head + 9 + keep).payload.payload.info + for label, edit in edits: + pairs = edit(list(cut.settings.items(multi=True))) + with self.subTest(captured=keep, edit=label): + built = {} + for name, info in (('cut', cut), ('whole', whole)): + data = info.to_dict() + data['settings'] = OrderedMultiDict(pairs) + built[name] = HTTPv2.from_data(data) + # The cut frame's record still cuts it back to the octets it captured. + self.assertEqual(built['cut'].info.length, built['whole'].info.length) + self.assertEqual(built['cut'].info.length, 6 * len(pairs)) + self.assertEqual(built['cut'].data, built['whole'].data[:9 + keep]) + def test_a_dict_with_no_length_computes_it(self) -> None: from pcapkit.const.http.frame import Frame from pcapkit.const.http.setting import Setting diff --git a/tests/protocols/internet/test_igmp_unit.py b/tests/protocols/internet/test_igmp_unit.py index c846ea65f6..cd54f4ddcd 100644 --- a/tests/protocols/internet/test_igmp_unit.py +++ b/tests/protocols/internet/test_igmp_unit.py @@ -1227,8 +1227,8 @@ def test_report_cut_inside_a_record_header(self) -> None: def test_report_cut_inside_a_source_list(self) -> None: """Cut inside the first record's second source: the source reads as far - as it was captured, and the second record the count declares reads as - zeros, as a zero-filled record of no sources (#2686).""" + as it was captured, and the second record the count declares, of which + no octet was captured, is not made up of zeros (#2756).""" from pcapkit.protocols.internet.igmpv3.protocol import IGMPv3 from pcapkit.protocols.misc.no_payload.protocol import NoPayload @@ -1243,7 +1243,7 @@ def test_report_cut_inside_a_source_list(self) -> None: self.assertEqual( [(int(record.record_type), str(record.multicast_address), tuple(map(str, record.sources))) for record in igmp.info.records], - [(1, '239.1.1.1', ('10.0.0.1', '10.0.0.0')), (0, '0.0.0.0', ())], + [(1, '239.1.1.1', ('10.0.0.1', '10.0.0.0'))], ) self.assertIsInstance(igmp.payload, NoPayload) self.assert_rebuilds(cut, parsed) diff --git a/tests/protocols/internet/test_igmpv3_framed_record_cut_2756_unit.py b/tests/protocols/internet/test_igmpv3_framed_record_cut_2756_unit.py new file mode 100644 index 0000000000..f0f5a5bff6 --- /dev/null +++ b/tests/protocols/internet/test_igmpv3_framed_record_cut_2756_unit.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +"""An IGMPv3 Report behind IPv4 and cut inside its records reads no record past the cut. + +GitHub issue #2756: an IPv4 datagram whose capture ends inside an IGMPv3 Report's group +records, its total length intact, parsed a record the capture never held. Behind a frame the +record list counts none (#2711), and its length is what IPv4 leaves, so its octets past the +cut were zero-filled to it (#2686) and read on as records: an ``Unassigned`` record of no +sources, with ``SchemaWarning: packet length < 0`` for what the zeros left. The list ends with +the data instead, as it does for a Report parsed on its own (#2711, #2749). + +The Report below is cut at every offset from the end of its fixed fields to inside its last +record, and wrapped in an IPv4 datagram declaring all of it. It must hold the records its +octets begin, the one the cut lands inside read as far as it was captured, read +``TRUNCATED`` with its short-read record and quietly, read as the same octets parsed on +their own, and rebuild as captured, the datagram and the Report alike. A whole datagram, and +the Report parsed on its own, are the controls. + +The messages are built here; this module is unit-tier and reads no capture file. + +""" +from __future__ import annotations + +import contextlib +import logging +import unittest +import warnings +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.foundation import _roundtrip as wire + +if TYPE_CHECKING: + from typing import Any, Iterator + +#: Test tier (#1958): hand-picked parse -> rebuild round trips. +__tier__ = 'ALL' + + +def record(kind: int, sources: int, aux: int = 0) -> bytes: + """A record of type ``kind`` for 239.1.1.1, with ``sources`` sources and ``aux`` words of + auxiliary data (RFC 3376, section 4.2.4).""" + return (bytes([kind, aux]) + sources.to_bytes(2, 'big') + bytes([239, 1, 1, 1]) + + bytes([10, 0, 0, 1]) * sources + b'AUX!' * aux) + + +def report(*records: bytes) -> bytes: + """An IGMPv3 Report of ``records``, checksummed.""" + rest = len(records).to_bytes(2, 'big') + b''.join(records) + head = b'\x22\x00' + return head + wire.checksum(head + bytes(4) + rest).to_bytes(2, 'big') + b'\x00\x00' + rest + + +#: MODE_IS_INCLUDE with two sources; MODE_IS_EXCLUDE with one source and a word of auxiliary +#: data; and ALLOW_NEW_SOURCES with two sources and two words of it. +RECORDS = (record(1, 2), record(2, 1, aux=1), record(5, 2, aux=2)) +#: The Report itself. +MESSAGE = report(*RECORDS) +#: Where each record starts in the message. +STARTS = tuple(8 + sum(map(len, RECORDS[:index])) for index in range(len(RECORDS))) + + +def begun(keep: int) -> int: + """How many records the first ``keep`` octets of the message begin.""" + return sum(1 for start in STARTS if start < keep) + + +class _Logged(logging.Handler): + """Collect what the ``pcapkit`` logger reports.""" + + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages = [] # type: list[str] + + def emit(self, record: 'logging.LogRecord') -> None: # pylint: disable=redefined-outer-name + self.messages.append(record.getMessage()) + + +@contextlib.contextmanager +def _reported() -> 'Iterator[list[str]]': + """Collect what the block reports, on the logger and the :mod:`warnings` channel.""" + handler = _Logged() + logger = logging.getLogger('pcapkit') + logger.addHandler(handler) + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + yield handler.messages + handler.messages.extend(f'{item.category.__name__}: {item.message}' for item in caught) + finally: + logger.removeHandler(handler) + + +class FramedReportCutTests(unittest.TestCase): + """A Report behind IPv4 and cut inside its records holds the records it begins.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + from pcapkit.corekit.checksum.internet import ChecksumStatus + from pcapkit.protocols.internet.igmpv3.protocol import IGMPv3 + from pcapkit.protocols.internet.ipv4.protocol import IPv4 + from pcapkit.utilities import config + + self.IGMPv3, self.IPv4, self.Status = IGMPv3, IPv4, ChecksumStatus + patch = mock.patch.object(config, 'CHECKSUM_VERIFY', True) + patch.start() + self.addCleanup(patch.stop) + + @staticmethod + def records(layer: 'Any') -> 'list[tuple[int, int, tuple[str, ...], bytes]]': + """``layer``'s records, each as its type, source count, sources and auxiliary data.""" + return [(int(item.record_type), item.num_sources, tuple(map(str, item.sources)), item.aux_data) + for item in layer.info.records] + + def framed(self, keep: int) -> 'tuple[bytes, Any, list[str]]': + """The datagram cut ``keep`` octets into the message, its parse, and what it reported.""" + packet = wire.ipv4(MESSAGE, proto=2) + raw = packet[:len(packet) - len(MESSAGE) + keep] + with _reported() as reported: + ipv4 = self.IPv4(raw) + return raw, ipv4, reported + + def test_the_cuts_reach_every_record_and_boundary(self) -> None: + self.assertEqual(STARTS, (8, 24, 40)) + self.assertEqual(len(MESSAGE), 64) + + def test_a_cut_holds_the_records_it_begins(self) -> None: + whole = self.records(self.IGMPv3(MESSAGE)) + for keep in range(8, len(MESSAGE)): + with self.subTest(captured=keep): + _, ipv4, reported = self.framed(keep) + igmp = ipv4.payload + self.assertIsInstance(igmp, self.IGMPv3) + self.assertEqual(reported, []) + self.assertEqual(igmp.info.get('__short_read__'), ('body', keep - 1)) + self.assertIs(igmp.info.checksum_status, self.Status.TRUNCATED) + self.assertEqual(igmp.info.num_group_records, len(RECORDS)) + records = self.records(igmp) + self.assertEqual(len(records), begun(keep)) + self.assertEqual(records[:-1], whole[:begun(keep) - 1] if records else []) + + def test_a_record_the_cut_lands_inside_reads_as_captured(self) -> None: + # One to three octets into the second record: its type was captured, and its + # Number of Sources reads as zero. + for keep in range(STARTS[1] + 1, STARTS[1] + 4): + with self.subTest(captured=keep): + _, ipv4, _ = self.framed(keep) + self.assertEqual([item[:3] for item in self.records(ipv4.payload)], + [(1, 2, ('10.0.0.1', '10.0.0.1')), (2, 0, ())]) + + def test_a_framed_cut_reads_as_its_octets_on_their_own(self) -> None: + for keep in range(8, len(MESSAGE)): + _, ipv4, _ = self.framed(keep) + for sized in (False, True): + with self.subTest(captured=keep, sized=sized), _reported(): + alone = self.IGMPv3(MESSAGE[:keep], keep) if sized else self.IGMPv3(MESSAGE[:keep]) + self.assertEqual(self.records(ipv4.payload), self.records(alone)) + + def test_a_cut_rebuilds_as_captured(self) -> None: + for keep in range(8, len(MESSAGE)): + raw, ipv4, _ = self.framed(keep) + igmp = ipv4.payload + for path, data, cls, octets in (('datagram', ipv4.info, self.IPv4, raw), + ('datagram dict', ipv4.info.to_dict(), self.IPv4, raw), + ('report', igmp.info, self.IGMPv3, MESSAGE[:keep])): + with self.subTest(captured=keep, path=path): + with _reported() as reported: + rebuilt = cls.from_data(data) + self.assertEqual((rebuilt.data, reported), (octets, [])) + + def test_a_whole_datagram_reads_every_record(self) -> None: + packet = wire.ipv4(MESSAGE, proto=2) + with _reported() as reported: + ipv4 = self.IPv4(packet) + igmp = ipv4.payload + self.assertEqual(reported, []) + self.assertIsNone(igmp.info.get('__short_read__')) + self.assertIs(igmp.info.checksum_status, self.Status.VALID) + self.assertEqual(self.records(igmp), [ + (1, 2, ('10.0.0.1', '10.0.0.1'), b''), + (2, 1, ('10.0.0.1',), b'AUX!'), + (5, 2, ('10.0.0.1', '10.0.0.1'), b'AUX!AUX!'), + ]) + self.assertEqual(self.IPv4.from_data(ipv4.info).data, packet) + + def test_a_report_on_its_own_is_unchanged(self) -> None: + # The count sizes the records of a Report parsed on its own (#2711), which a cut + # inside them reads as quietly as before (#2749). + for keep in range(8, len(MESSAGE)): + for sized in (False, True): + with self.subTest(captured=keep, sized=sized): + with _reported() as reported: + alone = self.IGMPv3(MESSAGE[:keep], keep) if sized else self.IGMPv3(MESSAGE[:keep]) + self.assertEqual(reported, []) + self.assertEqual(alone.info.get('__short_read__'), ('body', keep - 1)) + self.assertIs(alone.info.checksum_status, self.Status.TRUNCATED) + self.assertEqual(len(alone.info.records), begun(keep)) + self.assertEqual(self.IGMPv3.from_data(alone.info).data, MESSAGE[:keep]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/test_list_no_phantom_item_2756_unit.py b/tests/protocols/test_list_no_phantom_item_2756_unit.py new file mode 100644 index 0000000000..4dc8615268 --- /dev/null +++ b/tests/protocols/test_list_no_phantom_item_2756_unit.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +"""A list of schema items its length runs past the data reads no item past the data. + +GitHub issue #2756: an IGMPv3 Report behind IPv4, cut inside its records, parsed a record +the capture never held. A :class:`~pcapkit.corekit.fields.collections.ListField` of schema +items the data ends inside zero-fills its octets to its length (#2686), so the item the +data ends inside reads its missing octets as zeros, as any field does; but its items were +read on to that length, so every item past the data was made of the zeros alone. The list +ends with the data instead. + +The sweep reads two minimal items of every list of schema items under +:mod:`pcapkit.protocols`, found and built as :file:`tests/protocols/test_list_item_cut_2749_unit.py` +finds and builds them, behind a header field that sizes both, at every cut. Each cut is +refused with an in-library error, or holds the items its octets begin, no more, and packs +back to the octets read. Behind the zero-fill, the octets read are those present, the zeros +the items leave unread taken back. + +Each test imports :mod:`pcapkit` itself, after +:func:`~tests._support.reimport_once_per_class`, and builds its own octets. + +""" +from __future__ import annotations + +import unittest +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.protocols.test_list_item_cut_2749_unit import LISTS, NEGATIVE, REFUSED, holders, item_lists, read + +#: Test tier (#1958): a sweep over every list of schema items. +__tier__ = 'LATEST' + +#: Item schemas two of which are not a list the sweep can read, and why. +SKIP = { + 'internet.mh:CGAParameter': 'a CGA Parameters option carries one CGA Parameters structure (RFC 4866), ' + 'whose extensions run to the end of the option', +} + + +class NoItemPastTheDataTests(unittest.TestCase): + """A list its length runs past the data ends with the data.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_a_cut_list_holds_the_items_it_begins(self) -> None: + from tests.corekit.test_schema_roundtrip_unit import build, label_of + + read_any = set() + for _, item in item_lists(): + name = label_of(item.schema) + if name in SKIP: + continue + _, raw, ctx = build(item.schema) + _, sized = holders(item) + whole = raw * 2 + for keep in range(len(whole)): + data = len(whole).to_bytes(2, 'big') + whole[:keep] + with self.subTest(item=name, captured=keep): + parsed, caught = read(sized, data, 2 + len(whole), ctx) + if parsed is None: + continue + read_any.add(name) + self.assertEqual(len(parsed.items), -(-keep // len(raw))) + self.assertEqual([text for text in caught if NEGATIVE in text], []) + self.assertEqual(parsed.pack(dict(ctx)), data) + self.assertEqual(read_any, set(LISTS.values()) - set(REFUSED) - set(SKIP)) + + def test_a_whole_list_is_the_control(self) -> None: + from tests.corekit.test_schema_roundtrip_unit import build, label_of + + for _, item in item_lists(): + if label_of(item.schema) in SKIP: + continue + _, raw, ctx = build(item.schema) + _, sized = holders(item) + data = (2 * len(raw)).to_bytes(2, 'big') + raw * 2 + with self.subTest(item=label_of(item.schema)): + parsed, caught = read(sized, data, len(data), ctx) + self.assertEqual(caught, []) + self.assertIsNone(parsed.__dict__.get('__short_read__')) + self.assertEqual((len(parsed.items), parsed.pack(dict(ctx))), (2, data)) + + def test_the_octets_read_are_those_present(self) -> None: + from pcapkit.corekit.fields.collections import ListField + from pcapkit.corekit.fields.field import _zero_pad_budget + from pcapkit.corekit.fields.misc import SchemaField + from pcapkit.corekit.fields.numbers import UInt16Field, UInt32Field + from pcapkit.protocols.schema import Schema, schema_final + from pcapkit.utilities import config + + @schema_final + class Pair(Schema): + id = UInt16Field() + value = UInt32Field() + + field = ListField(length=18, item_type=SchemaField(length=6, schema=Pair)) + pairs = bytes(range(1, 19)) + for keep in range(len(pairs)): + with self.subTest(captured=keep), mock.patch.object(config, 'ZERO_PAD_BUDGET_RATIO', -1), \ + _zero_pad_budget() as ledger: + parsed = field.unpack(pairs[:keep], {}) + self.assertEqual(len(parsed), -(-keep // 6)) + self.assertEqual(ledger, [keep, len(pairs) - keep]) + + +if __name__ == '__main__': + unittest.main()