fix(corekit): read a capture cut inside an option area as a short read (#2694)

docs/source/contributing/conventions/index.rst

@@ -30,8 +30,8 @@ House Conventions
    * :ref:`test-coverage` -- what the test suite has to prove;
    * :ref:`utilities-or-corekit` -- which of :mod:`pcapkit.utilities` and
      :mod:`pcapkit.corekit` a support module belongs in, where a context
-     variable lives, and where a schema helper used outside the schema layer or
-     outside its own protocol lives;
+     variable lives, where a schema helper used outside the schema layer or
+     outside its own protocol lives, and what reads a capture cut;
    * :ref:`reassembly-integration` -- :mod:`pcapkit.foundation.reassembly` and what a
      new engine there needs from :mod:`pcapkit.toolkit` and
      :mod:`pcapkit.foundation.analysis`;

docs/source/contributing/conventions/utilities-or-corekit.rst

@@ -72,6 +72,18 @@ header schema framework, which stays :mod:`pcapkit.protocols.schema` and its
 :class:`~pcapkit.protocols.schema.schema.Schema`; it holds only the helpers shared
 across the layer.
 
+**A capture cut belongs to the built-in short read of**
+:class:`~pcapkit.protocols.schema.schema.Schema`. :meth:`Schema.unpack
+<pcapkit.protocols.schema.schema.Schema.unpack>` records the field the data ends
+inside, through ``_note_short_read``, as ``__short_read__``;
+:meth:`~pcapkit.protocols.schema.schema.Schema._cut_short_read` packs back only the
+octets read; and :func:`~pcapkit.corekit.schema.shortread.keep_short_read` and
+:func:`~pcapkit.corekit.schema.shortread.replay_short_read` carry the record through
+``info`` and cut a rebuild back by it. A protocol reads and rebuilds a cut header
+through them, never through an ``unpack`` override of its own or a length callable
+clamped to the octets present, which hides the cut from the record. Ruled in review
+of :pr:`2682` (:issue:`2680`), and applied to option areas on :issue:`2694`.
+
 **A helper in one protocol's own schema module moves there too once code outside that
 protocol uses it and it is generic enough to reuse**, so that another implementor can
 find it; a helper only its own protocol calls stays beside the schema. Ruled on

docs/source/pcapkit/protocols/internet/mh.rst

@@ -988,6 +988,7 @@ Auxiliary Functions
 .. autofunction:: pcapkit.protocols.schema.internet.mh.bid_address_selector
 .. autofunction:: pcapkit.protocols.schema.internet.mh.br_code_selector
 .. autofunction:: pcapkit.protocols.schema.internet.mh.cga_public_key_length
+.. autofunction:: pcapkit.protocols.schema.internet.mh.item_list_length
 .. autofunction:: pcapkit.protocols.schema.internet.mh.dmnp_prefix_selector
 .. autofunction:: pcapkit.protocols.schema.internet.mh.fb_code_selector
 .. autofunction:: pcapkit.protocols.schema.internet.mh.lma_address_selector

pcapkit/corekit/fields/collections.py

@@ -445,19 +445,40 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> 'list
             ``packet`` as ``__option_padding__`` so that the next fields can
             be aware of it.
 
+            A ``__capture_cut__`` key in ``packet``, false, says that the data
+            ends at the length the enclosing layer declares for it, so that an
+            area the data ends inside runs past that length rather than being
+            cut short by the capture, as
+            :class:`~pcapkit.protocols.internet.ipv6.IPv6` tells the extension
+            headers it walks (:issue:`2694`). Absent, the data may end at a cut.
+
         Raises:
             FieldValueError: If an option consumes nothing from ``buffer``, since
                 the loop below has then no way to get past it.
             ProtocolError: If an option that begins with octets present reads
-                past the end of the data, unless :meth:`_accepts_short_read`
-                accepts it.
+                past the end of the data, unless the data ends inside the area,
+                the option's fields fit it and :meth:`_accepts_cut` keeps it, or
+                :meth:`_accepts_short_read` accepts it. Or if the area runs past
+                the end of the data where ``packet`` gives ``__capture_cut__``
+                as false, the data ending at the length the enclosing layer
+                declares rather than at a cut.
 
         .. versionchanged:: 1.5.0
 
            An option that begins with octets present and reads past the end of
            the data raises :exc:`~pcapkit.utilities.exceptions.ProtocolError`,
            where it was parsed from zero-filled octets (:issue:`1325`).
 
+        .. versionchanged:: 1.6.0
+
+           Where the data ends inside the area, as a capture cut short leaves
+           it, the list ends with the data, and an option it ends inside is
+           kept if its fields fit the area, so :meth:`Schema.unpack
+           <pcapkit.protocols.schema.schema.Schema.unpack>` records the cut as
+           a short read rather than either error being raised
+           (:issue:`2694`). See :meth:`_accepts_cut`, and ``__capture_cut__``
+           above.
+
         """
         length = self._length
         if isinstance(buffer, bytes):
@@ -529,6 +550,24 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> 'list
         # nothing at all and is left to the end-of-option-list and progress
         # handling below, which is what an option area declared longer than its
         # octets relies on.
+        #
+        # Unless the data ends inside the area: fewer octets are left than it
+        # declares, as a capture cut short leaves them. An option the cut lands
+        # inside, or in front of, is not one running past its area, and is read
+        # as the short read :meth:`Schema.unpack` records (#2694). An enclosing
+        # layer that knows the data ends at the length it declares, not at a cut,
+        # says so with ``__capture_cut__`` false in ``packet``, and the area then
+        # runs past that length.
+        if isinstance(buffer, bytes):
+            present = len(buffer)
+        else:
+            try:
+                present = file.seek(0, io.SEEK_END) - start
+                file.seek(start, io.SEEK_SET)
+            except (OSError, ValueError):  # a stream that cannot seek to its end counts as whole
+                present = self._length
+        short = present < self._length
+        cut = short and packet.get('__capture_cut__', True)
         tracker = file = _ShortReadTracker(file)  # type: ignore[assignment]
 
         temp = []  # type: list[_TS]
@@ -547,14 +586,35 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> 'list
                 byte = file.read(field.length)
                 code = field.unpack(byte, packet.copy())
                 consumed = len(byte)
+
+            # NOTE: The data ends where this option would begin, inside the area: the
+            # cut leaves no option to read, so the list ends with the data, and
+            # :meth:`Schema.unpack` records the cut from the rest of the area left
+            # as ``option_padding`` (#2694). Where no cut ends it, the area runs
+            # past the data. A registry whose end-of-option-list code is 0 reads its
+            # exhausted type as that code, and stops at it below.
+            if not consumed and short and code != self._eool:
+                if cut:
+                    break
+                raise ProtocolError(
+                    f'Field {self.name} has an option area that runs past the end of '
+                    f'the data: {self._length} octet(s) declared, with {present} present'
+                )
             schema = self._registry[code]
 
             # rewind to the beginning of the option
             file.seek(-consumed, io.SEEK_CUR)
 
             # unpack option using option schema
             data = schema.unpack(file, length, packet)  # type: ignore[call-arg,misc,var-annotated]
-            if tracker.short and consumed > 0 and not self._accepts_short_read(data):
+
+            # NOTE: An option the cut lands inside stands when the fields it
+            # declares fit the area, as ``__length__`` left at zero or more shows,
+            # and it read the octets there (#2694). One whose fields reach past
+            # the area runs past both (#1325).
+            stands = (cut and packet['__length__'] >= 0 and file.tell() > offset
+                      and self._accepts_cut(data))
+            if tracker.short and consumed > 0 and not stands and not self._accepts_short_read(data):
                 raise ProtocolError(
                     f'Field {self.name} has an option that runs past the end of '
                     f'the data: {code!r} at offset {offset - start} of '
@@ -604,7 +664,8 @@ def _accepts_short_read(self, option: '_TS') -> 'bool':  # pylint: disable=unuse
         """Whether an option that read past the end of the data may stand.
 
         Args:
-            option: Option just unpacked, which read short.
+            option: Option just unpacked, which read short and ran past its area
+                or past the data at the end of the area.
 
         Returns:
             :data:`False`, so that :meth:`unpack` refuses the option
@@ -617,3 +678,20 @@ def _accepts_short_read(self, option: '_TS') -> 'bool':  # pylint: disable=unuse
 
         """
         return False
+
+    def _accepts_cut(self, option: '_TS') -> 'bool':  # pylint: disable=unused-argument
+        """Whether an option the data ends inside, inside its area, may stand.
+
+        Args:
+            option: Option just unpacked, which read short where the data ends
+                short of the area declared, and whose fields fit the area.
+
+        Returns:
+            :data:`True`, so that :meth:`unpack` keeps the option, as the short
+            read :meth:`Schema.unpack <pcapkit.protocols.schema.schema.Schema.unpack>`
+            records (:issue:`2694`). A list that keeps such an area as the octets
+            captured instead may override this, as
+            :class:`~pcapkit.protocols.schema.misc.pcapng.OptionAreaField` does.
+
+        """
+        return True

pcapkit/protocols/internet/hopopt.py

@@ -92,7 +92,7 @@
     TunnelEncapsulationLimitOption as Schema_TunnelEncapsulationLimitOption
 from pcapkit.protocols.schema.internet.hopopt import UnassignedOption as Schema_UnassignedOption
 from pcapkit.protocols.schema.schema import Schema
-from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall
+from pcapkit.utilities.exceptions import BaseError, CaughtByCaller, ProtocolError, UnsupportedCall
 from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn
 
 if TYPE_CHECKING:
@@ -395,14 +395,21 @@ def read(self, length: 'Optional[int]' = None, *, extension: 'bool' = False, **k
     def make(self,
              next: 'Enum_TransType | int' = Enum_TransType.UDP,
              options: 'Optional[list[Schema_Option | tuple[Enum_Option | int, dict[str, Any]] | bytes] | Option]' = None,  # pylint: disable=line-too-long
-             payload: 'bytes | ProtocolBase | Schema' = b'',
+             payload: 'bytes | ProtocolBase | Schema' = b'', *,
+             len: 'Optional[int]' = None,  # pylint: disable=redefined-builtin
              **kwargs: 'Any') -> 'Schema_HOPOPT':
         """Make (construct) packet data.
 
         Args:
             next: Next header type.
             options: Hop-by-Hop Options.
             payload: Payload of current protocol.
+            len: ``Hdr Ext Len`` to declare, rather than the one the options
+                fill. A header the capture cut short declares one its parsed
+                options do not fill, which :meth:`from_data` passes on, so that
+                the rebuild declares it too (:issue:`2694`). Options that fill
+                less than it declares are not padded out to it, so the header
+                then declares more octets than it packs, as a cut one does.
             **kwargs: Arbitrary keyword arguments.
 
         Returns:
@@ -412,7 +419,12 @@ def make(self,
             ProtocolError: If ``next``, or the type of an option, is a name, a
                 :obj:`bool`, or a member or code that does not fit in one
                 octet; or if the options make a header too long for its
-                one-octet ``Hdr Ext Len``.
+                one-octet ``Hdr Ext Len``, or longer than ``len`` declares.
+
+        .. versionchanged:: 1.6.0
+
+           Takes ``len``, the ``Hdr Ext Len`` of a header the capture cut short
+           (:issue:`2694`).
 
         """
         next_value = self._make_index(next, Enum_TransType, bits=8, name='next header')
@@ -428,6 +440,11 @@ def make(self,
         # NOTE: ``_make_hopopt_options`` has aligned the header, so this division
         # is exact and rounding up is unnecessary.
         length = self._make_uint((total_length - 6) // 8, bits=8, name='Hdr Ext Len')
+        if len is not None:
+            length = self._make_uint(len, bits=8, name='Hdr Ext Len')
+            if length * 8 + 6 < total_length:
+                raise ProtocolError(f'{self.alias}: Hdr Ext Len {length} declares fewer octets '
+                                    f'than the options take ({total_length})')
 
         return Schema_HOPOPT(
             next=next_value,
@@ -538,10 +555,13 @@ def _make_data(cls, data: 'Data_HOPOPT') -> 'dict[str, Any]':  # type: ignore[ov
             Key-value pairs for protocol construction.
 
         """
+        # NOTE: A header the capture cut short declares an area its parsed options
+        # do not fill, and is rebuilt with the Hdr Ext Len it declared (#2694).
         return {
             'next': data.next,
             'options': data.options,
             'payload': cls._make_payload(data),
+            'len': data.length // 8 - 1 if data.get('__short_read__') is not None else None,
         }
 
     def _read_opt_type(self, kind: 'int') -> 'tuple[int, bool]':
@@ -609,7 +629,16 @@ def _read_hopopt_options(self, length: 'int') -> 'Option':
 
         Raises:
             ProtocolError: If the parsed options do not total exactly ``length``
-                octets.
+                octets, unless the capture cut the header short, as its short
+                read records. Or if the capture cut an option short and its
+                parser refuses the zeros it lacks, or its maker does not write
+                the octets captured back.
+
+        .. versionchanged:: 1.6.0
+
+           A header the capture cut short reads as the short read it is, and an
+           option it cuts as its parser reads the zeros it lacks, rather than
+           being refused (:issue:`2694`).
 
         """
         counter = 0                   # length of read options
@@ -623,16 +652,34 @@ def _read_hopopt_options(self, length: 'int') -> 'Option':
                 meth_name = f'_read_opt_{name}'
                 meth = cast('OptionParser',
                             getattr(self, meth_name, self._read_opt_none))
+                maker = cast('OptionConstructor',
+                             getattr(self, f'_make_opt_{name}', self._make_opt_none))
+            else:
+                meth, maker = name
+
+            # NOTE: The capture cut this option short, as its short read records, so
+            # the octets it lacks read as zeros. It stands if its parser takes those
+            # zeros and its maker writes the octets captured back (#2694); otherwise
+            # it is refused, as an option running past the data is (#1325).
+            if '__short_read__' in schema.__dict__:
+                try:
+                    with CaughtByCaller(BaseError):
+                        data = meth(schema, options=options)
+                        made = maker(dscp, data).pack()[:len(schema)]
+                except BaseError as error:
+                    raise ProtocolError(f'{self.alias}: [OptNo {dscp}] option cut short: {error}') from error
+                if made != bytes(schema):
+                    raise ProtocolError(f'{self.alias}: [OptNo {dscp}] option cut short does not '
+                                        'rebuild as captured')
             else:
-                meth = name[0]
-            data = meth(schema, options=options)
+                data = meth(schema, options=options)
 
             # record option data
             options.add(dscp, data)
             counter += len(schema)
 
-        # check threshold
-        if counter != length:
+        # check threshold, which a header the capture cut short falls short of (#2694)
+        if counter != length and '__short_read__' not in self.__header__.__dict__:
             raise ProtocolError('HOPOPT: invalid format')
         return options
 

pcapkit/protocols/internet/ipv6.py

@@ -582,13 +582,18 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None,
             #     proto = None
             #     break
 
+            # NOTE: The Payload Length left bounds this header. With every octet of
+            # it present, a header longer than that runs past it, malformed and
+            # not cut by the capture, which its option area is told (#2694).
+            context = dict(packet or {}, __capture_cut__=len(payload) < raw_len)
+
             # make protocol name
             # NOTE: Counted after every header walked, as it would be were each
             # dissected by the one before it. ESP dissects its decrypted
             # payload even when walked, and those layers count on from here.
             token = frame_depth.set(depth + len(_protos))
             try:
-                next_ = self._import_next_layer(proto, packet=packet, version=6, extension=True,
+                next_ = self._import_next_layer(proto, packet=context, version=6, extension=True,
                                                 payload=payload)  # type: ignore[misc,call-arg]
             finally:
                 frame_depth.reset(token)