diff --git a/docs/source/pcapkit/foundation/engines/index.rst b/docs/source/pcapkit/foundation/engines/index.rst index 6e587f827..c4a7bdcf8 100644 --- a/docs/source/pcapkit/foundation/engines/index.rst +++ b/docs/source/pcapkit/foundation/engines/index.rst @@ -76,10 +76,7 @@ offers, and the ones that cannot say so rather than quietly doing less: | Engine | Gap, and how it is surfaced | +=================================================================+===============================================================+ | :class:`~pcapkit.foundation.engines.pyshark.PyShark` | no reassembly -- disabled with an | -| | :class:`~pcapkit.utilities.warnings.AttributeWarning`, and | -| | :program:`tshark`'s own IP reassembly is turned off, so a | -| | fragmented datagram is read frame by frame, as the built-in | -| | engine reads it | +| | :class:`~pcapkit.utilities.warnings.AttributeWarning` | +-----------------------------------------------------------------+---------------------------------------------------------------+ | :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` | no protocol dissection at all, hence no reassembly and no | | | flow tracing -- both disabled with an | diff --git a/pcapkit/foundation/engines/pyshark.py b/pcapkit/foundation/engines/pyshark.py index eb65ae769..48dfe392b 100644 --- a/pcapkit/foundation/engines/pyshark.py +++ b/pcapkit/foundation/engines/pyshark.py @@ -10,143 +10,31 @@ .. _PyShark: https://kiminewt.github.io/pyshark """ -import json -import os -import subprocess import sys -import tempfile from typing import TYPE_CHECKING, cast -from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType from pcapkit.foundation.engines.engine import EngineBase -from pcapkit.foundation.engines.pcap import PCAP as PCAP_Engine -from pcapkit.foundation.engines.pcapng import PCAPNG as PCAPNG_Engine from pcapkit.foundation.reassembly import ReassemblyManager -from pcapkit.protocols.misc.pcap.header import Header -from pcapkit.utilities.exceptions import FormatError, stacklevel +from pcapkit.utilities.exceptions import stacklevel from pcapkit.utilities.logging import get_logger -from pcapkit.utilities.warnings import AttributeWarning, PySharkWarning, warn +from pcapkit.utilities.warnings import AttributeWarning, warn __all__ = ['PyShark'] if TYPE_CHECKING: - from typing import IO, Generator, Literal, Optional, Union + from typing import Optional from pyshark.capture.file_capture import FileCapture from pyshark.packet.layers.base import BaseLayer from pyshark.packet.packet import Packet as PySharkPacket from pcapkit.foundation.extraction import Extractor - from pcapkit.toolkit.pyshark import Captured #: logging.Logger: Module-level logger, a child of the package-wide #: :data:`pcapkit.utilities.logging.logger`. logger = get_logger(__name__) -def _ek_record(line: 'bytes') -> 'Optional[Union[tuple[int, bytes], str]]': - """Parse a line of :program:`tshark`'s ``-T ek -x -J frame`` output. - - Args: - line: One line of the output. - - Returns: - :data:`None` for an index line, which each packet line follows; for a - packet line, its ``frame.number`` and its frame's captured octets, the - ``frame_raw`` that ``-x`` adds; otherwise, a phrase saying the line - cannot be parsed. - - """ - try: - record = json.loads(line) - if isinstance(record, dict) and set(record) == {'index'}: - return None - layers = record['layers'] - number, octets = layers['frame']['frame_frame_number'], layers['frame_raw'] - if isinstance(number, list): - number = number[0] - if isinstance(octets, list): - octets = octets[0] - return int(number), bytes.fromhex(octets) - except (ValueError, LookupError, TypeError): # not JSON, a field missing, or not text - return f'gave output it cannot parse: {bytes(line[:80])!r}' - - -def _linktypes(path: 'str') -> 'Generator[int, None, None]': - """Read the link type of each packet of a capture file, in order, as the default engine reads it. - - Args: - path: The PCAP or PCAP-NG file. - - Yields: - The link type the default engine dissects each packet by. A PCAP file - has one, in its global header. A PCAP-NG packet takes that of the - interface it was captured on, among the Interface Description Blocks of - its own section: an Enhanced or obsolete Packet Block names it by ID, and - a Simple Packet Block is the first's. These three are the packets the - default engine counts as frames; every other block is skipped. - - Raises: - FormatError: If a PCAP-NG block is of no byte order a Section Header - Block declares, too short for its fields, or a packet block naming - an interface its section does not describe. A block cut short by - the end of the file ends the packets instead. - - Note: - The file is opened at the first packet asked for, and closed at its end, - or when the generator is closed. - - """ - # length of the field after a block's type and length that names its link - # type (IDB) or interface (EPB, obsolete Packet Block), by block type - fields = { - int(Enum_BlockType.Interface_Description_Block): 2, - int(Enum_BlockType.Enhanced_Packet_Block): 4, - int(Enum_BlockType.Packet_Block): 2, - } # type: dict[int, int] - - with open(path, 'rb') as file: - if file.read(4) in PCAP_Engine.MAGIC_NUMBER: - file.seek(0) - linktype = int(Header(file).protocol) - while True: - yield linktype - - file.seek(0) - byteorder = None # type: Optional[Literal['big', 'little']] - interfaces = [] # type: list[int] - while True: - head = file.read(8) - if head[:4] == PCAPNG_Engine.MAGIC_NUMBER[0]: # a palindrome, the same in either byte order - head += file.read(4) - if len(head) < 12: - return - byteorder = PCAPNG_Engine.BYTE_ORDER_MAGIC.get(head[8:]) - interfaces = [] # an interface ID counts the IDBs of its own section - elif len(head) < 8: - return - if byteorder is None: - raise FormatError(f'PCAP-NG: block of no known byte order: {head!r}') - kind, length = int.from_bytes(head[:4], byteorder), int.from_bytes(head[4:8], byteorder) - size = fields.get(kind, 0) - if length < len(head) + size + 4: # the type, length, field, and trailing length - raise FormatError(f'PCAP-NG: [Block {kind}] invalid block length: {length}') - - field = file.read(size) - file.seek(length - len(head) - size - 4, os.SEEK_CUR) - if len(field) < size or len(file.read(4)) < 4: - return - - if kind == Enum_BlockType.Interface_Description_Block: - interfaces.append(int.from_bytes(field, byteorder)) - elif kind in (Enum_BlockType.Enhanced_Packet_Block, Enum_BlockType.Packet_Block, - Enum_BlockType.Simple_Packet_Block): - interface = int.from_bytes(field, byteorder) # 0 for a Simple Packet Block, which has none - if interface >= len(interfaces): - raise FormatError(f'PCAP-NG: [Block {kind}] invalid interface ID: {interface}') - yield interfaces[interface] - - def _frame_layer(packet: 'PySharkPacket') -> 'Optional[BaseLayer]': """The ``frame`` layer of a `PyShark`_ packet, or :data:`None` when the record has none. @@ -200,21 +88,6 @@ class PyShark(EngineBase['PySharkPacket']): _expkg: 'pyshark' #: Engine extraction temporary storage. _extmp: 'FileCapture' - #: The second :program:`tshark` pass :meth:`_captured` reads captured - #: octets from, once it is first asked for some. - _exraw: 'Optional[subprocess.Popen[bytes]]' - #: Where that pass writes its standard error. - _exraw_err: 'Optional[IO[bytes]]' - #: The frame number and octets :meth:`_captured` last read from it. - _exraw_held: 'Optional[tuple[int, bytes]]' - #: The link type of each packet of the capture file (see - #: :func:`_linktypes`), which :meth:`_captured` reads, once it is first - #: asked for one. - _exlink: 'Optional[Generator[int, None, None]]' - #: The count and link type of the packet :meth:`_captured` last read from it. - _exlink_held: 'Optional[tuple[int, int]]' - #: Whether reading either has failed, so :meth:`_captured` reads no more. - _exraw_failed: 'bool' ########################################################################## # Defaults. @@ -233,29 +106,6 @@ class PyShark(EngineBase['PySharkPacket']): #: event loop exists instead of quietly creating one. PYTHON_CEILING = (3, 14) - #: :program:`tshark` preferences :meth:`run` overrides, through - #: ``pyshark``'s ``override_prefs``: IPv4 and IPv6 reassembly off. tshark - #: reassembles fragments by default and dissects what the datagram carries - #: at the frame that completes it, so TCP was traced at the last fragment. - #: The default engine reads each frame as captured, TCP from a first - #: fragment (:issue:`1612`), and pcapkit's own reassembly is disabled on - #: this engine. - #: - #: Turning tshark's off also changes its TCP analysis, which is not - #: confined to fragments. Measured with tshark 4.6.9: tshark sizes a - #: fragmented segment from its first fragment alone, so its sequence - #: analysis sees a gap after it. Later frames of that stream, fragments or - #: not, then gain ``tcp.analysis.lost_segment``, and their other - #: ``tcp.analysis`` fields, ``tcp.completeness``, ``tcp.time_delta`` and - #: ``tcp.stream.*.contiguity_count`` change. Those fields reach the frame - #: records :func:`~pcapkit.toolkit.pyshark.packet2dict` builds. But - #: :func:`~pcapkit.toolkit.pyshark.tcp_traceflow` reads none of them, so - #: flow tracing is unaffected. The 23 captures under - #: :file:`examples/captures` carry no fragmented TCP. On them, every field - #: of every frame is the same either way, except the first and last - #: fragments of the one fragmented UDP datagram in :file:`ipv6.pcap`. - OVERRIDE_PREFS = {'ip.defragment': 'FALSE', 'ipv6.defragment': 'FALSE'} # type: dict[str, str] - ########################################################################## # Class methods. ########################################################################## @@ -352,12 +202,6 @@ def __init__(self, extractor: 'Extractor') -> 'None': self._expkg = pyshark self._extmp = cast('FileCapture', None) - self._exraw = None - self._exraw_err = None - self._exraw_held = None - self._exlink = None - self._exlink_held = None - self._exraw_failed = False super().__init__(extractor) @@ -370,8 +214,7 @@ def run(self) -> 'None': This method assigns :attr:`self._expkg ` as :mod:`pyshark` and :attr:`self._extmp ` - as an iterator from :class:`pyshark.FileCapture`, with - :program:`tshark`'s IP reassembly off (:attr:`OVERRIDE_PREFS`). + as an iterator from :class:`pyshark.FileCapture`. The global header is parsed and dumped first, by :meth:`self.extractor.record_header `. @@ -424,8 +267,7 @@ def run(self) -> 'None': # extract & analyse file logger.debug('pyshark: opening %s', ext._ifnm) - self._extmp = self._expkg.FileCapture(ext._ifnm, keep_packets=False, - override_prefs=dict(self.OVERRIDE_PREFS)) + self._extmp = self._expkg.FileCapture(ext._ifnm, keep_packets=False) def read_frame(self) -> 'PySharkPacket': """Read frames with PyShark engine. @@ -469,7 +311,7 @@ def read_frame(self) -> 'PySharkPacket': # trace flows if ext._flag_t: if ext._tcp: - data_tf_tcp = tcp_traceflow(packet, count=ext._frnum, captured=self._captured) + data_tf_tcp = tcp_traceflow(packet, count=ext._frnum) if data_tf_tcp is not None: ext._trace.tcp(data_tf_tcp) @@ -481,197 +323,11 @@ def read_frame(self) -> 'PySharkPacket': # return frame record return packet - def _octets_command(self) -> 'list[str]': - """The command line of the second :program:`tshark` pass :meth:`_captured` reads. - - Returns: - :program:`tshark`, found as `PyShark`_ finds it, reading the capture - with the preferences :meth:`run` gives `PyShark`_ - (:attr:`OVERRIDE_PREFS`) and no name resolution, and writing the - frames :data:`~pcapkit.toolkit.pyshark.OCTETS_FILTER` matches as - Elasticsearch JSON, one line each, with their captured octets - (``-x``) and no protocol's fields but the frame's (``-J frame``). - - .. _PyShark: https://kiminewt.github.io/pyshark - - """ - from pyshark.tshark.tshark import get_process_path # isort:skip - - from pcapkit.toolkit.pyshark import OCTETS_FILTER # isort:skip - - command = [get_process_path(), '-l', '-n', '-r', self._extractor._ifnm, '-Y', OCTETS_FILTER, - '-T', 'ek', '-x', '-J', 'frame'] - for name, value in self.OVERRIDE_PREFS.items(): - command += ['-o', f'{name}:{value}'] - return command - - def _read_octets(self, number: 'int') -> 'Union[bytes, str]': - """Read the captured octets of a frame from the second :program:`tshark` pass. - - Args: - number: The frame's ``frame.number``. - - Returns: - The octets, or a phrase saying why there are none. - - """ - if self._exraw is None: - self._exraw_err = tempfile.TemporaryFile() # pylint: disable=consider-using-with - self._exraw = subprocess.Popen(self._octets_command(), stdin=subprocess.DEVNULL, # pylint: disable=consider-using-with - stdout=subprocess.PIPE, stderr=self._exraw_err) - held = self._exraw_held - while held is None or held[0] < number: - line = cast('IO[bytes]', self._exraw.stdout).readline() - if not line: - status = self._exraw.wait() - if not status: - return f'ended before frame {number}' - error = cast('IO[bytes]', self._exraw_err) - error.seek(0) - last = error.read().decode('utf-8', 'replace').strip().splitlines()[-1:] - return f'exited with status {status}' + ''.join(f' ({text})' for text in last) - record = _ek_record(line) - if isinstance(record, str): - return record - if record is not None: - held = self._exraw_held = record - if held[0] != number: - return f'has no frame {number}' - return held[1] - - def _read_linktype(self, count: 'int') -> 'Union[int, str]': - """Read the link type the default engine dissects a frame by, from the capture file. - - Args: - count: The frame's count, as the default engine numbers frames. - - Returns: - The link type, or a phrase saying why there is none. - - """ - if self._exlink is None: - self._exlink = _linktypes(self._extractor._ifnm) - held = self._exlink_held - while held is None or held[0] < count: - linktype = next(self._exlink, None) - if linktype is None: - return f'has no packet {count}' - held = self._exlink_held = ((0 if held is None else held[0]) + 1, linktype) - if held[0] != count: - return f'was read past packet {count}' - return held[1] - - def _captured(self, packet: 'PySharkPacket') -> 'Optional[Captured]': - """Read a frame's link type and octets as the default engine reads them, which the PDML :meth:`run` reads lacks. - - :func:`~pcapkit.toolkit.pyshark.tcp_traceflow` asks for one of a frame - :data:`~pcapkit.toolkit.pyshark.OCTETS_FILTER` matches only: one tshark - dissects TCP in (:issue:`1639`), one with an AH or IPv6 extension - header in IPv4 (:issue:`1625`), or one with more 802.1Q tags than - tshark dissects (:issue:`1609`). - - Its link type is the one the default engine dissects it by, read from - the capture file (see :func:`_linktypes`) for the frame of this engine's - count, which numbers frames as the default engine does. tshark's - ``frame.encap_type`` cannot say which it is, as tshark reads several link - types as one (see :data:`~pcapkit.toolkit.pyshark.ENCAP_TYPE_TO_LINKTYPE`). - - Its captured octets come from a second :program:`tshark` pass over the - capture (see :meth:`_octets_command`), started on the first request. - Both passes number frames alike and deliver them in order, so the second - is read forward to the frame asked for, and the last frame it read is - kept for the next request. The capture file is read forward likewise. A - capture with no such frame reads neither. - - Args: - packet: PyShark packet of the first pass. - - Returns: - The frame's link type and captured octets; :data:`None` once either - cannot be read. - - Warns: - PySharkWarning: When the frame cannot be read: the second pass - cannot be started, exits with an error, gives output that cannot - be parsed, or lacks the frame asked for; or the capture file - cannot be read for the frame's link type. Both are then stopped, - and this and every later frame is traced as tshark dissects it, - as before :issue:`1625` and :issue:`1639`, which can disagree - with the default engine. - - Note: - The second pass is a process of this engine's own rather than a - second :class:`pyshark.FileCapture`, for two reasons measured with - `PyShark`_ 0.6 and :program:`tshark` 4.6.9. Its JSON parser reads - ``frame.time_epoch``, which a Simple Packet Block lacks, and raises - :exc:`KeyError`. And each capture it creates sets a new - :class:`asyncio.SafeChildWatcher` (``Capture._setup_eventloop``), so - the process of the first is left unreaped once a second exists. - - .. _PyShark: https://kiminewt.github.io/pyshark - - """ - from pcapkit.toolkit.pyshark import Captured # isort:skip - - if self._exraw_failed: - return None - frnum = self._extractor._frnum - frames = ('a frame with an AH or IPv6 extension header in IPv4, with TCP, or with more 802.1Q tags ' - 'than tshark dissects,') - source = f'the capture file, read for the link type of {frames}' - try: - linktype = self._read_linktype(frnum) - if isinstance(linktype, str): - reason = linktype - else: - source = f'the second tshark pass, which reads the captured octets of {frames}' - octets = self._read_octets(int(packet.number)) - if isinstance(octets, bytes): - return Captured(linktype, octets) - reason = octets - except Exception as exc: # pylint: disable=broad-except - reason = f'failed: {type(exc).__name__}: {exc}' - - self._exraw_failed = True - self._stop_captured() - warn(f'Frame {frnum}: {source} {reason}; this and later such frames are traced as tshark ' - 'dissects them, which can disagree with the default engine', - PySharkWarning, stacklevel=stacklevel()) - return None - - def _stop_captured(self) -> 'None': - """Stop reading frames as :meth:`_captured` reads them. - - The second :program:`tshark` pass, if it was started, is stopped and - reaped, and the capture file, if it was opened, closed. - - """ - process, self._exraw = self._exraw, None - error, self._exraw_err = self._exraw_err, None - linktypes, self._exlink = self._exlink, None - try: - if process is not None: - if process.poll() is None: - process.kill() - process.wait() - cast('IO[bytes]', process.stdout).close() - finally: - try: - if error is not None: - error.close() - finally: - if linktypes is not None: - linktypes.close() - def close(self) -> 'None': """Close engine. This method is to be used for closing the engine instance. It is to - close the engine instance after the extraction process is finished, - and what :meth:`_captured` reads frames from, if anything. + close the engine instance after the extraction process is finished. """ - try: - self._extmp.close() - finally: - self._stop_captured() + self._extmp.close() diff --git a/pcapkit/toolkit/pyshark.py b/pcapkit/toolkit/pyshark.py index 2c06c19dd..e28c5037c 100644 --- a/pcapkit/toolkit/pyshark.py +++ b/pcapkit/toolkit/pyshark.py @@ -18,26 +18,18 @@ """ import decimal import ipaddress -from typing import TYPE_CHECKING, NamedTuple +from typing import TYPE_CHECKING -from pcapkit.const.reg.ethertype import EtherType as Enum_EtherType from pcapkit.const.reg.linktype import LinkType as Enum_LinkType -from pcapkit.const.reg.transtype import TransType as Enum_TransType from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet -from pcapkit.protocols.link.loopback import family_byteorder -from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT -from pcapkit.toolkit.pypcapfile import (IP_TUNNELS, IPV4_EXTENSION_HEADERS, IPV4_HEADER_LEN, - TCP_MIN_HEADER_LEN, VLAN_TAG_LEN, VLAN_TAGS, - _default_accepts, _walk_upper_layer) from pcapkit.utilities.exceptions import MissingKeyError, ProtocolError if TYPE_CHECKING: - from ipaddress import IPv4Address, IPv6Address - from typing import Any, Callable, Optional + from typing import Any from pyshark.packet.packet import Packet -__all__ = ['packet2dict', 'tcp_traceflow', 'Captured', 'ENCAP_TYPE_TO_LINKTYPE', 'FILTER_NAME_TO_LINKTYPE'] +__all__ = ['packet2dict', 'tcp_traceflow', 'ENCAP_TYPE_TO_LINKTYPE', 'FILTER_NAME_TO_LINKTYPE'] #: ``frame.encap_type`` -- Wireshark's internal ``WTAP_ENCAP_*`` number, which :mod:`pyshark` #: exposes as ``packet.frame_info.encap_type`` -- to the matching @@ -69,14 +61,6 @@ #: 152 of them are below; ``frame.encap_type`` 32 (``editcap -T hhdlc``, DLT 121) is left out #: because :class:`LinkType` has no member for 121. An encapsulation absent from this table raises #: rather than resolving to a near-miss DLT. -#: -#: In the read direction, one key can come from several DLTs. Measured with ``tshark`` 4.6.9 over -#: every PCAP-NG link type from 0 to 65535 (bar 144 and 177, under which it rejected the test -#: frame), ``frame.encap_type`` 7 is read from DLTs 12, 14 and 101, 4 from 9 and 50, 10 from 11 -#: and 100, 11 from 16, 18, 19 and 106, 28 from 104 and 112, and 38 from 13 and 109; each other -#: key of this table from one DLT. So where :func:`tcp_traceflow` reads a frame as the default -#: engine does, it takes the link type from the capture file rather than from this table -#: (:issue:`1625`). ENCAP_TYPE_TO_LINKTYPE = { 1: Enum_LinkType.ETHERNET, # ether 2: Enum_LinkType.IEEE802_5, # tr @@ -338,78 +322,6 @@ #: on tshark 4.6.9); ``'1'``/``'0'`` is the spelling this module first parsed (#1514). _BOOLEAN_FIELD = {'1': True, '0': False, 'True': True, 'False': False} # type: dict[str, bool] -#: IPv4 protocol numbers that make :func:`tcp_traceflow` read a frame from its -#: captured octets though tshark dissects no TCP segment in it (see -#: :func:`_extension_header_carrier`): the AH and IPv6 extension headers. The -#: default engine reads past those of the -#: :data:`~pcapkit.toolkit.pypcapfile.IPV4_EXTENSION_HEADERS` by rules of its -#: own, and past Shim6 not at all, as its parser refuses IPv4; tshark dissects -#: the TCP behind any of them by its own rules. -OCTETS_PROTOCOLS = IPV4_EXTENSION_HEADERS | {Enum_TransType.Shim6} - -#: tshark display filter matching every frame :func:`tcp_traceflow` asks the -#: captured octets of: one tshark dissects TCP in (see :func:`_traces`), one -#: with an IPv4 header whose protocol is one of the :data:`OCTETS_PROTOCOLS`, or -#: one whose dissection stops in a stack of 802.1Q tags (see -#: :func:`_stops_in_vlan_tags`). It matches every ``tcp`` and ``vlan`` layer and -#: ``ip.proto`` field of a frame, so it matches all of those frames and maybe -#: others. The members are joined with ``||`` rather than written as a set, as -#: tshark 4.6.9 refuses a set whose members are separated by spaces only. -OCTETS_FILTER = ' || '.join(['tcp', *(f'ip.proto == {int(protocol)}' for protocol in sorted(OCTETS_PROTOCOLS)), - 'vlan']) - -#: Length of an Ethernet header: two MAC addresses and the EtherType. -ETHERNET_HEADER_LEN = 14 - -#: Length of a BSD loopback header: the address family. -LOOPBACK_HEADER_LEN = 4 - -#: EtherTypes of the IP layers the default engine dissects over Ethernet, each -#: mapped to its IP version (see :attr:`Link.__proto__ -#: `). -ETHERTYPE_IP_VERSIONS = { - Enum_EtherType.Internet_Protocol_version_4: 4, - Enum_EtherType.Internet_Protocol_version_6: 6, -} # type: dict[int, int] - -#: Address families of the IP layers the default engine dissects over BSD -#: loopback, each mapped to its IP version (see :attr:`Loopback.__proto__ -#: `). -LOOPBACK_IP_VERSIONS = {2: 4, 24: 6, 28: 6, 30: 6} # type: dict[int, int] - - -class Captured(NamedTuple): - """A frame as the default engine reads it from the capture file (:issue:`1625`).""" - - #: Link type the default engine dissects the frame by: the PCAP file's, or - #: that of the PCAP-NG interface the frame was captured on. - linktype: 'int' - #: The frame's captured octets. - octets: 'bytes' - - -class _Segment(NamedTuple): - """The fields of a TCP segment that a flow tracing record takes from it.""" - - #: Source address of the IP layer carrying the segment. - src: 'IPv4Address | IPv6Address' - #: Destination address of that IP layer. - dst: 'IPv4Address | IPv6Address' - #: TCP source port. - srcport: 'int' - #: TCP destination port. - dstport: 'int' - #: TCP sequence number, as on the wire. - seq: 'int' - #: TCP acknowledgement number, as on the wire. - ack: 'int' - #: TCP synchronise (SYN) flag. - syn: 'bool' - #: TCP finish (FIN) flag. - fin: 'bool' - #: TCP reset (RST) flag. - rst: 'bool' - def _parse_flag(value: 'Any', name: 'str') -> 'bool': """Parse a boolean PyShark field, such as ``tcp.flags_syn``. @@ -601,450 +513,28 @@ def _has_fixed_header(tcp: 'Any') -> 'bool': return 'urgent_pointer' in tcp.field_names -def _header_addresses(ip: 'Any') -> 'Optional[tuple[IPv4Address | IPv6Address, IPv4Address | IPv6Address]]': - """Read the addresses of an IP header from tshark's dissection of it. +def _has_addresses(ip: 'Any') -> 'bool': + """Whether tshark reported both addresses of a PyShark IP layer. Args: ip: PyShark ``ip`` or ``ipv6`` layer. Returns: - The header's Source and Destination Address fields, as the default - engine keys a segment by them; or :data:`None` if tshark reports no - source or no destination address. - - Note: - Behind an LSRR or SSRR option whose route is not yet used up - (:rfc:`791#section-3.1`), tshark reports the route's last address as - ``ip.dst``, at its place in the option, and the header's as - ``ip.cur_rt``, which it reports nowhere else. Where the option's route - data is no whole number of addresses, it reports no ``ip.dst`` at all - ("Suboption would go past end of option"); the default engine's IPv4 - parser rejects such an option, so reads no TCP behind it. Both measured - with tshark 4.6.9 over every length and pointer of an LSRR and SSRR - option an IPv4 header holds (:issue:`1671`). - - """ - try: - src, dst = ip.src, ip.dst - except AttributeError: # what PyShark raises for a field tshark does not report - return None - return ipaddress.ip_address(str(src)), ipaddress.ip_address(str(getattr(ip, 'cur_rt', dst))) - - -def _from_layers(packet: 'Packet') -> 'Optional[_Segment]': - """Read the TCP segment of a PyShark packet from tshark's dissection of it. - - Args: - packet: PyShark packet. - - Returns: - The packet's first TCP layer, if it has a whole fixed header (see - :func:`_has_fixed_header`), keyed by the addresses of the IP header - that carries it (see :func:`_header_addresses`) -- in an IP-in-IP - tunnel, the innermost (see :func:`_carrier`); else :data:`None`. - - Raises: - ProtocolError: If a TCP flag field is not a boolean tshark spelling. - - """ - ip, tcp = _carrier(packet) - if ip is None or tcp is None or not _has_fixed_header(tcp): - return None - addresses = _header_addresses(ip) - if addresses is None: - return None - return _Segment( - src=addresses[0], - dst=addresses[1], - srcport=int(tcp.srcport), - dstport=int(tcp.dstport), - seq=_absolute(tcp, 'seq'), - ack=_absolute(tcp, 'ack'), - syn=_parse_flag(tcp.flags_syn, 'tcp.flags_syn'), - fin=_parse_flag(tcp.flags_fin, 'tcp.flags_fin'), - rst=_parse_flag(tcp.flags_reset, 'tcp.flags_reset'), - ) - - -def _traces(packet: 'Packet') -> 'bool': - """Whether tshark's dissection of a PyShark packet has a TCP segment to trace. - - Args: - packet: PyShark packet. - - Returns: - Whether its first TCP layer has a whole fixed header (see - :func:`_has_fixed_header`) and an IP layer carries it (see - :func:`_carrier`). - - """ - ip, tcp = _carrier(packet) - return ip is not None and tcp is not None and _has_fixed_header(tcp) - - -def _stops_in_vlan_tags(packet: 'Packet') -> 'bool': - """Whether tshark's dissection of a PyShark packet stops in a stack of 802.1Q tags. - - Args: - packet: PyShark packet. - - Returns: - Whether it has a ``vlan`` layer and no ``ip`` or ``ipv6`` layer. + Whether the layer carries its ``src`` and ``dst`` fields. Note: - tshark dissects at most 20 802.1Q tags of a frame, 802.1ad tags between - them or not, and nothing past the 21st ("Too many nested VLAN tags", - measured with tshark 4.6.9). The default engine dissects every one of - the :data:`~pcapkit.toolkit.pypcapfile.VLAN_TAGS` and the IP packet - behind them, up to :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` - layers of the frame (:issue:`1609`). - - """ - names = {layer.layer_name.lower() for layer in packet.layers} - return 'vlan' in names and not names & {'ip', 'ipv6'} - - -def _extension_header_carrier(packet: 'Packet') -> 'Any': - """Find the IPv4 layer of a PyShark packet whose protocol names an AH or IPv6 extension header. - - Args: - packet: PyShark packet. - - Returns: - The first ``ip`` layer above the packet's first ``tcp`` layer, or of a - packet with none, whose ``ip.proto`` is one of the - :data:`OCTETS_PROTOCOLS`; else :data:`None`. + tshark reports no ``ip.dst`` behind an LSRR or SSRR option whose route + data is no whole number of addresses ("Suboption would go past end of + option"), measured with tshark 4.6.9. Reading it raised a bare + :exc:`AttributeError` out of :func:`tcp_traceflow`, which ended the + extraction (:issue:`1671`). """ - for layer in packet.layers: - name = layer.layer_name.lower() - if name == 'tcp': - break - if name == 'ip': - try: - protocol = int(str(getattr(layer, 'proto', ''))) - except ValueError: - continue - if protocol in OCTETS_PROTOCOLS: - return layer - return None - + names = ip.field_names + return 'src' in names and 'dst' in names -def _link_type(packet: 'Packet', frame: 'Any') -> 'Enum_LinkType': - """Resolve the link type of a PyShark packet's frame. - - Args: - packet: PyShark packet. - frame: Its ``frame`` layer (see :func:`_frame_layer`). - Returns: - The link type ``frame.encap_type`` names, through - :data:`ENCAP_TYPE_TO_LINKTYPE`, or the PDML root layer's filter name, - through :data:`FILTER_NAME_TO_LINKTYPE`, where the frame layer has no - such field. - - Raises: - MissingKeyError: If the link type cannot be resolved. - - """ - # NOTE: the link type is keyed on ``frame.encap_type`` -- Wireshark's - # internal ``WTAP_ENCAP_*`` number -- through - # ``ENCAP_TYPE_TO_LINKTYPE`` (module level, above), and only on the - # PDML root layer's *filter* name when the frame layer has no such - # field. That name cannot do the job on its own: one filter name - # serves several DLTs, so keying on it would answer ``RAW`` (101) for a - # ``rawip6`` capture and ``NULL`` (0) for a ``DLT_LOOP`` one -- valid - # DLTs, wrong ones, and silent (#843). ``frame.encap_type`` - # distinguishes them; see that table's comment for the measurements. - # - # PyShark reports every field as ``LayerFieldsContainer``, a ``str`` - # subclass, so the number arrives as text and needs ``int()``; and a - # field the frame layer does not carry raises ``AttributeError`` out - # of ``pyshark.packet.layers.base.BaseLayer.__getattr__``, which is - # what the ``getattr`` default absorbs here. - # - # Neither path substitutes a DLT on a miss. A lookup raises on an - # unresolvable key rather than minting one (#775), and - # NULL and RAW are genuine DLTs -- BSD loopback and raw IP framing, - # respectively -- each meant to go with its own handler protocol - # class, so neither is an honest stand-in for "unknown link type". - # The bare ``KeyError``/``ValueError`` is re-raised as - # :exc:`~pcapkit.utilities.exceptions.MissingKeyError` -- this - # package's own house exception for a lookup miss -- rather than - # letting it escape the public :func:`tcp_traceflow`. - # - # The root layer is the first layer that is not the frame layer. - encap_type = getattr(frame, 'encap_type', None) - if encap_type is None: - name = [layer for layer in packet.layers if layer is not frame][0].layer_name - try: - return FILTER_NAME_TO_LINKTYPE[name.lower()] - except KeyError: - raise MissingKeyError(name) from None - try: - return ENCAP_TYPE_TO_LINKTYPE[int(encap_type)] - except (KeyError, ValueError): - raise MissingKeyError('frame.encap_type=%s' % encap_type) from None - - -def _network_start(linktype: 'int', octets: 'bytes') -> 'Optional[tuple[int, int, int]]': - """Find the outermost IP layer the default engine dissects in a frame, from its octets. - - Args: - linktype: The link type the default engine dissects the frame by, the - capture file's (see :class:`Captured`). - octets: The frame's captured octets. - - Returns: - The layer's IP version, where its header starts in ``octets``, and its - place in the frame, as the default engine counts layers towards - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`: ``1`` where the - frame is the IP packet, else ``2``, and one more for each VLAN tag, - which is a layer of its own (:issue:`1609`). :data:`None` if the - default engine dissects no IP layer right after the link layer. - - Note: - The default engine dissects a frame by its link type - (:attr:`Frame.__proto__ `). - That is the capture file's, not the one tshark's ``frame.encap_type`` - names (see :func:`_link_type`): tshark reads link types 12 and 14, raw - IP on some BSDs, as it reads ``RAW`` (101), and the default engine - leaves them raw. - Over Ethernet, it reads the EtherTypes in :data:`ETHERTYPE_IP_VERSIONS` - after the 14-octet header and every one of the - :data:`~pcapkit.toolkit.pypcapfile.VLAN_TAGS` behind it; over BSD - loopback (``NULL`` and ``LOOP``), the address families in - :data:`LOOPBACK_IP_VERSIONS` after the 4-octet family, in the byte - order :func:`~pcapkit.protocols.link.loopback.family_byteorder` gives. - ``IPV4`` and ``IPV6`` frames are IP, and a ``RAW`` one is the IP - version its first octet gives, 4 or 6. It leaves every other link type, - EtherType and family raw, and a frame cut short of its link header. - - """ - if linktype == Enum_LinkType.ETHERNET: - if len(octets) < ETHERNET_HEADER_LEN: - return None - ethertype, start = int.from_bytes(octets[12:14], 'big'), ETHERNET_HEADER_LEN - while ethertype in VLAN_TAGS: - if len(octets) < start + VLAN_TAG_LEN: - return None - ethertype, start = int.from_bytes(octets[start + 2:start + 4], 'big'), start + VLAN_TAG_LEN - version, depth = ETHERTYPE_IP_VERSIONS.get(ethertype), 2 + (start - ETHERNET_HEADER_LEN) // VLAN_TAG_LEN - elif linktype in (Enum_LinkType.NULL, Enum_LinkType.LOOP): - if len(octets) < LOOPBACK_HEADER_LEN: - return None - family, start, depth = octets[:LOOPBACK_HEADER_LEN], LOOPBACK_HEADER_LEN, 2 - version = LOOPBACK_IP_VERSIONS.get(int.from_bytes(family, family_byteorder(family, linktype))) - elif linktype == Enum_LinkType.RAW: - if not octets: - return None - version, start, depth = octets[0] >> 4, 0, 1 - elif linktype in (Enum_LinkType.IPV4, Enum_LinkType.IPV6): - version, start, depth = (4 if linktype == Enum_LinkType.IPV4 else 6), 0, 1 - else: - return None - if version is None or version not in (4, 6): - return None - return version, start, depth - - -def _network(packet: 'Packet', captured: 'Captured') -> 'Optional[tuple[int, bytes, int]]': - """Find the outermost IP packet the default engine dissects in a PyShark packet's frame. - - Args: - packet: PyShark packet. - captured: The frame, as the default engine reads it from the capture file. - - Returns: - The packet's IP version, its octets on, to the end of the frame, and - its place in the frame, as :func:`_network_start` finds it by the - frame's link type, if tshark dissected it too: the packet's first - ``ip`` or ``ipv6`` layer is of that version, and tshark reports its - version field there, the layer's source address in the header's Source - Address field, and its destination address in the octets where tshark - reports that field (see below). Or if tshark dissected no IP layer, as - its dissection stopped in the VLAN tags before it (see - :func:`_stops_in_vlan_tags`). Else :data:`None`: the default engine - dissects no IP packet right after the link layer, or tshark dissected - none there, e.g. as it reads an IPv6 packet over link type ``IPV4`` - (228) as IPv6, or dissected that layer out of other octets, e.g. - another frame's. - - Note: - The destination address is matched where tshark reports it, not in - the header's Destination Address field: with an LSRR or SSRR option - whose route is not yet used up (:rfc:`791#section-3.1`), tshark - reports the route's last address as ``ip.dst``, at its place in the - option, and the header's as ``ip.cur_rt``, measured with tshark 4.6.9. - The default engine keys the segment by the header's, as the octets do. - - """ - linktype, octets = captured - found = _network_start(linktype, octets) - if found is None: - return None - version, start, depth = found - - name = 'ip' if version == 4 else 'ipv6' - ip = next((layer for layer in packet.layers if layer.layer_name.lower() in ('ip', 'ipv6')), None) - if ip is None and _stops_in_vlan_tags(packet): - return version, octets[start:], depth - if ip is None or ip.layer_name.lower() != name: - return None - try: - position, target = int(ip.version.pos), int(ip.dst.pos) - source, destination = bytes.fromhex(str(ip.src.raw_value)), bytes.fromhex(str(ip.dst.raw_value)) - except (AttributeError, TypeError, ValueError): - return None - size = 4 if version == 4 else 16 - where = start + (12 if version == 4 else 8) # the source address - if (position != start or len(source) != size or len(destination) != size - or octets[where:where + size] != source - or target < start or octets[target:target + size] != destination): - return None - return version, octets[start:], depth - - -def _from_octets(version: 'int', data: 'bytes', depth: 'int') -> 'Optional[_Segment]': - """Read the TCP segment the default engine reads in an IP packet, from its octets. - - Args: - version: IP version of the header ``data`` starts with, ``4`` or ``6``. - data: Raw octets from the IP header on, to the end of the frame. - depth: The IP header's place in its frame, its first layer being ``1`` - (see :func:`_network_start`). - - Returns: - The segment, keyed by the IP layer that carries it -- in an IP-in-IP - tunnel, the innermost -- or :data:`None` if the default engine reads no - TCP segment there. Only IPv4 and IPv6 in IP (the - :data:`~pcapkit.toolkit.pypcapfile.IP_TUNNELS`) are followed: no other - tunnel, such as GRE or VXLAN, as the default engine dissects none. - - Every IPv4 header on the way must have options the default engine's - IPv4 parser accepts (see - :func:`~pcapkit.toolkit.pypcapfile._default_accepts`), and each IP - packet is followed past its headers by - :func:`~pcapkit.toolkit.pypcapfile._walk_upper_layer`: an AH or IPv6 - extension header in IPv4 by the default engine's own parser, and the - extension headers of IPv6, up to - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of - them in a row; none in a later fragment. No layer is read past - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the - frame, each IP header, AH and IPv6 extension header one of them - (:issue:`1609`). A TCP header must be at least - :data:`~pcapkit.toolkit.pypcapfile.TCP_MIN_HEADER_LEN` octets long, as - its Data Offset gives it, with that many captured. - - Behind an AH or IPv6 extension header in IPv4, the default engine reads - TCP only as its TCP parser does, so the whole of that header must be - there, with options the parser accepts. Directly over IP, it reads a - header the parser rejects out of the raw octets left behind - (:func:`pcapkit.toolkit.pcap.tcp_segment`), the rule - :func:`_has_fixed_header` applies to tshark's own dissection. - - Note: - These are the rules the pypcapfile, dpkt and scapy engines apply to - the same octets (:func:`pcapkit.toolkit.pypcapfile._transport`, - :issue:`1606`). - - """ - while True: - if depth > FRAME_LAYER_LIMIT: - return None - if version == 4 and len(data) >= IPV4_HEADER_LEN: - ihl = (data[0] & 0x0F) * 4 - if IPV4_HEADER_LEN < ihl <= len(data) and not _default_accepts('IPv4', data[:ihl]): - return None - upper = _walk_upper_layer(version, data, depth) - if upper is None: - return None - if upper[0] not in IP_TUNNELS: - break - version, data, depth = IP_TUNNELS[upper[0]], upper[1], upper[2] - - protocol, segment, _ = upper - if protocol != Enum_TransType.TCP or len(segment) < TCP_MIN_HEADER_LEN: - return None - header = (segment[12] >> 4) * 4 # the Data Offset, in octets - if header < TCP_MIN_HEADER_LEN: - return None - if version == 4 and data[9] != Enum_TransType.TCP and (header > len(segment) or ( - header > TCP_MIN_HEADER_LEN and not _default_accepts('TCP', segment[:header]))): - return None - - src, dst = (data[12:16], data[16:20]) if version == 4 else (data[8:24], data[24:40]) - flags = segment[13] - return _Segment( - src=ipaddress.ip_address(src), - dst=ipaddress.ip_address(dst), - srcport=int.from_bytes(segment[0:2], 'big'), - dstport=int.from_bytes(segment[2:4], 'big'), - seq=int.from_bytes(segment[4:8], 'big'), - ack=int.from_bytes(segment[8:12], 'big'), - syn=bool(flags & 0x02), - fin=bool(flags & 0x01), - rst=bool(flags & 0x04), - ) - - -def _segment(packet: 'Packet', captured: 'Optional[Callable[[Packet], Optional[Captured]]]') -> 'Optional[_Segment]': - """Read the TCP segment of a PyShark packet that the default engine reads. - - Args: - packet: PyShark packet. - captured: Fetches a frame as the default engine reads it from the - capture file, or :data:`None` where it cannot be had (see - :func:`tcp_traceflow`). - - Returns: - If tshark's dissection has a segment (see :func:`_traces`), an IPv4 - layer whose protocol names an AH or IPv6 extension header (see - :func:`_extension_header_carrier`), or stops in a stack of 802.1Q tags - (see :func:`_stops_in_vlan_tags`), the segment read from the octets of - the frame's outermost IP packet on (see :func:`_network` and - :func:`_from_octets`), or :data:`None` where the default engine - dissects no IP packet after the frame's link layer or tshark dissects - none there. Otherwise, or where the frame cannot be had, the segment of - tshark's dissection (see :func:`_from_layers`). :data:`None` if there - is none. - - Raises: - ProtocolError: If a TCP flag field of tshark's dissection is not a - boolean tshark spelling. - - Note: - tshark dissects TCP inside encapsulations the default engine leaves - raw: link types 12 and 14, which it reads as raw IP (101), Linux SLL, - an EtherType such as MPLS or PPPoE, 802.3 LLC/SNAP, Cisco ISL, and - tunnels such as GRE, VXLAN and L2TP (:issue:`1639`). And it dissects - an AH or IPv6 extension header in IPv4 and what its Next Header names - by rules of its own, and nothing past a Mobility (135) or HIP (139) - header, whose payload it leaves out of the PDML that PyShark reads. - The default engine dispatches IPv4's protocol through the registry - IPv6 uses, reads TCP behind such a header only if its TCP parser - accepts it, and none behind Shim6 (140), whose parser refuses IPv4 - (:issue:`1625`). Reading the frame from its link layer on, as the - default engine does, stops where it stops: at a link type, EtherType or - tunnel it does not dissect, past - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` - extension headers in a row of an IPv6 packet, or past - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the - frame, which tshark reads on past; and it reads on past the 20 802.1Q - tags of a frame tshark stops at (:issue:`1609`). - - """ - if captured is not None and (_traces(packet) or _extension_header_carrier(packet) is not None - or _stops_in_vlan_tags(packet)): - frame = captured(packet) - if frame is not None: - network = _network(packet, frame) - return None if network is None else _from_octets(*network) - return _from_layers(packet) - - -def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1, - captured: 'Optional[Callable[[Packet], Optional[Captured]]]' = None) -> 'TF_TCP_Packet | None': +def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | None': """Trace packet flow for TCP. Args: @@ -1052,30 +542,15 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1, count: Packet index. If not provided, default to ``-1``. The engine passes its own frame count, not tshark's ``frame.number``, which also numbers records that are not packets (#1515). - captured: Fetches a frame as the default engine reads it from the - capture file, given its packet: its link type there and its - captured octets, neither of which the PDML carries; or - :data:`None` where it cannot. It is called only for a packet - :data:`OCTETS_FILTER` matches. The engine passes - :meth:`PyShark._captured - `; without - it, or where it returns :data:`None`, a packet is read as tshark - dissects it. Returns: Data for TCP flow tracing. * If the ``packet`` can be used for TCP flow tracing. A packet can be reassembled if it contains TCP layer with a whole fixed header (see - :func:`_has_fixed_header`), keyed by the addresses of the IP header that - carries it (see :func:`_header_addresses`) -- in an IP-in-IP tunnel, the - innermost (see :func:`_carrier`). Where - ``captured`` has the frame, it is instead the TCP the default engine - reads in the frame's octets, by the frame's link type in the capture - file, and none where the default engine reads none: behind a link - type, EtherType or tunnel it does not dissect, or past - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the - frame (see :func:`_segment`). + :func:`_has_fixed_header`), keyed by the IP layer that carries it -- in an + IP-in-IP tunnel, the innermost (see :func:`_carrier`) -- if tshark reports + both its addresses (see :func:`_has_addresses`). * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`. @@ -1089,16 +564,53 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1, :class:`pcapkit.foundation.traceflow.tcp.TCP` """ - segment = _segment(packet, captured) - if segment is not None: - - # NOTE: the frame layer is found by name, not taken as ``frame_info``, - # which is ``pkt_comment`` on a commented packet; ``_link_type`` says - # how the link type is keyed on it. + ip, tcp = _carrier(packet) + if ip is not None and tcp is not None and _has_fixed_header(tcp) and _has_addresses(ip): + + # NOTE: the link type is keyed on ``frame.encap_type`` -- Wireshark's + # internal ``WTAP_ENCAP_*`` number -- through + # ``ENCAP_TYPE_TO_LINKTYPE`` (module level, above), and only on the + # PDML root layer's *filter* name when the frame layer has no such + # field. That name cannot do the job on its own: one filter name + # serves several DLTs, so keying on it would answer ``RAW`` (101) for a + # ``rawip6`` capture and ``NULL`` (0) for a ``DLT_LOOP`` one -- valid + # DLTs, wrong ones, and silent (#843). ``frame.encap_type`` + # distinguishes them; see that table's comment for the measurements. + # + # PyShark reports every field as ``LayerFieldsContainer``, a ``str`` + # subclass, so the number arrives as text and needs ``int()``; and a + # field the frame layer does not carry raises ``AttributeError`` out + # of ``pyshark.packet.layers.base.BaseLayer.__getattr__``, which is + # what the ``getattr`` default absorbs here. + # + # Neither path substitutes a DLT on a miss. A lookup raises on an + # unresolvable key rather than minting one (#775), and + # NULL and RAW are genuine DLTs -- BSD loopback and raw IP framing, + # respectively -- each meant to go with its own handler protocol + # class, so neither is an honest stand-in for "unknown link type". + # The bare ``KeyError``/``ValueError`` is re-raised as + # :exc:`~pcapkit.utilities.exceptions.MissingKeyError` -- this + # package's own house exception for a lookup miss -- rather than + # letting it escape this public function. + # + # The frame layer is found by name, not taken as ``frame_info``, which + # is ``pkt_comment`` on a commented packet; the root layer is then the + # first layer that is not the frame layer. frame = _frame_layer(packet) if frame is None: raise ProtocolError(f'PyShark packet {packet.number} has no frame layer') - protocol = _link_type(packet, frame) + encap_type = getattr(frame, 'encap_type', None) + if encap_type is None: + name = [layer for layer in packet.layers if layer is not frame][0].layer_name + try: + protocol = FILTER_NAME_TO_LINKTYPE[name.lower()] + except KeyError: + raise MissingKeyError(name) from None + else: + try: + protocol = ENCAP_TYPE_TO_LINKTYPE[int(encap_type)] + except (KeyError, ValueError): + raise MissingKeyError('frame.encap_type=%s' % encap_type) from None # A Simple Packet Block carries no timestamp, so tshark reports no # ``frame.time_epoch`` for it; the PDML ``geninfo`` timestamp PyShark @@ -1112,21 +624,19 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1, protocol=protocol, # data link type index=count, # frame number frame=packet2dict(packet), # extracted packet - syn=segment.syn, # TCP synchronise (SYN) flag - fin=segment.fin, # TCP finish (FIN) flag - rst=segment.rst, # TCP reset (RST) flag - src=segment.src, # source IP - dst=segment.dst, # destination IP - srcport=segment.srcport, # TCP source port - dstport=segment.dstport, # TCP destination port + syn=_parse_flag(tcp.flags_syn, 'tcp.flags_syn'), # TCP synchronise (SYN) flag + fin=_parse_flag(tcp.flags_fin, 'tcp.flags_fin'), # TCP finish (FIN) flag + rst=_parse_flag(tcp.flags_reset, 'tcp.flags_reset'), # TCP reset (RST) flag + src=ipaddress.ip_address(ip.src), # source IP + dst=ipaddress.ip_address(ip.dst), # destination IP + srcport=int(tcp.srcport), # TCP source port + dstport=int(tcp.dstport), # TCP destination port timestamp=_parse_epoch(epoch), # timestamp - seq=segment.seq, # TCP sequence number - ack=segment.ack, # TCP acknowledgement number + seq=_absolute(tcp, 'seq'), # TCP sequence number + ack=_absolute(tcp, 'ack'), # TCP acknowledgement number # NOTE: PyShark reports dissected *fields*, not the octets behind # them, so there is no header or payload to hand over -- which is # the same reason this module carries no ``tcp_reassembly`` at all. - # The octets ``_segment`` reads are missing wherever the frame - # cannot be had, so are not handed over either. # ``Extractor`` refuses ``trace_analyse=True`` on this engine, so # nothing reads these two. header=b'', # unavailable diff --git a/tests/foundation/engines/test_runtime_engines.py b/tests/foundation/engines/test_runtime_engines.py index 534633665..a989cd67e 100644 --- a/tests/foundation/engines/test_runtime_engines.py +++ b/tests/foundation/engines/test_runtime_engines.py @@ -448,8 +448,7 @@ def close(self) -> None: engine = PyShark(extractor) with mock.patch.object(engine._expkg, 'FileCapture', return_value=capture) as file_capture: engine.run() - file_capture.assert_called_once_with('capture.pcap', keep_packets=False, - override_prefs=PyShark.OVERRIDE_PREFS) + file_capture.assert_called_once_with('capture.pcap', keep_packets=False) extractor.record_header.assert_called_once_with() self.assertFalse(extractor._flag_r) self.assertIsNone(extractor._reasm.ipv4) diff --git a/tests/toolkit/test_pyshark_encapsulated_tcp_1639_unit.py b/tests/toolkit/test_pyshark_encapsulated_tcp_1639_unit.py deleted file mode 100644 index 53a26b2db..000000000 --- a/tests/toolkit/test_pyshark_encapsulated_tcp_1639_unit.py +++ /dev/null @@ -1,599 +0,0 @@ -# -*- coding: utf-8 -*- -"""The pyshark engine traces no TCP inside an encapsulation the default engine leaves raw. C.f. #1639. - -tshark dissects TCP inside link types 12 and 14, which it reads as raw IP -(101), Linux SLL, EtherTypes such as MPLS and PPPoE, 802.3 LLC/SNAP and Cisco -ISL, and tunnels such as GRE, VXLAN and L2TP. The default engine leaves each of -these raw, so pyshark traced TCP it does not. A frame tshark finds TCP in is now -read from its octets by the default engine's rules, the path #1625 added for an -extension header in IPv4: from the outermost IP layer the default engine -dissects, by the link type in the capture file, through IP-in-IP alone. A frame -it reads no IP in, or one whose IP layer tshark dissects elsewhere, is not -traced; one whose octets cannot be had is traced as tshark dissects it, as -before. tshark's IP layer is matched to the octets by its addresses, the -destination where tshark reports it: behind an LSRR or SSRR option, that is -the route's last address, in the option. - -Every capture is built here, so the module is in the unit tier. The tests that -run tshark skip unless the pyshark engine runs, which needs ``pyshark``, the -``tshark`` binary and Python 3.13 or older; the rest use fakes. - -""" -from __future__ import annotations - -import importlib.util -import ipaddress -import os -import struct -import tempfile -import types -import unittest -import warnings -from typing import TYPE_CHECKING -from unittest import mock - -from tests._support import close_extractor, reimport_once_per_class -from tests.foundation import _roundtrip as wire - -if TYPE_CHECKING: - from typing import Any, Callable - -HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) - - -def tcp(number: int) -> bytes: - """A TCP segment carrying ``data`` on a port pair of its own.""" - return wire.tcp(b'data', seq=1000 + number, ack=1, sport=1000 + number) - - -def v4(payload: bytes, proto: int = 6, *, inner: bool = False, opts: bytes = b'') -> bytes: - """An IPv4 packet carrying ``payload`` as ``proto``, with options ``opts``; ``inner`` swaps the addresses.""" - packet = bytearray(wire.ipv4(payload, proto=proto, reverse=inner)) - if opts: - packet[0] = 0x40 | (5 + len(opts) // 4) - packet[2:4] = (len(packet) + len(opts)).to_bytes(2, 'big') - packet[20:20] = opts - packet[10:12] = b'\x00\x00' - packet[10:12] = wire.checksum(bytes(packet[:len(opts) + 20])).to_bytes(2, 'big') - return bytes(packet) - - -def v6(payload: bytes, nxt: int = 6, *, inner: bool = False) -> bytes: - """An IPv6 packet carrying ``payload`` as ``nxt``; ``inner`` swaps the addresses.""" - return wire.ipv6(payload, nxt=nxt, reverse=inner) - - -def eth(packet: bytes, ethertype: 'int | None' = None) -> bytes: - """An Ethernet frame carrying ``packet``, by default as its IP version's EtherType.""" - return wire.ethernet(packet, (0x86DD if packet[0] >> 4 == 6 else 0x0800) if ethertype is None else ethertype) - - -def gre(packet: bytes, ptype: 'int | None' = None) -> bytes: - """A GRE packet, with no options, carrying ``packet`` as ``ptype``, by default its IP version's.""" - if ptype is None: - ptype = 0x86DD if packet[0] >> 4 == 6 else 0x0800 - return b'\x00\x00' + ptype.to_bytes(2, 'big') + packet - - -def vxlan(frame: bytes) -> bytes: - """A UDP datagram to the VXLAN port, carrying the Ethernet ``frame`` with VNI 100.""" - return wire.udp(b'\x08\x00\x00\x00' + (100).to_bytes(3, 'big') + b'\x00' + frame, dport=4789) - - -def l2tp(packet: bytes) -> bytes: - """A UDP datagram to the L2TP port, carrying the IPv4 ``packet`` in PPP.""" - return wire.udp(b'\x00\x02\x00\x01\x00\x01\xff\x03\x00\x21' + packet, sport=1701, dport=1701) - - -def mpls(packet: bytes, labels: int = 1) -> bytes: - """``labels`` MPLS labels, the last the bottom of the stack, then ``packet``.""" - return b''.join(((16 + index) << 12 | (index == labels - 1) << 8 | 64).to_bytes(4, 'big') - for index in range(labels)) + packet - - -def pppoe(packet: bytes) -> bytes: - """A PPPoE session frame carrying ``packet`` in PPP.""" - proto = 0x0057 if packet[0] >> 4 == 6 else 0x0021 - return wire.ethernet(b'\x11\x00\x00\x01' + (2 + len(packet)).to_bytes(2, 'big') + proto.to_bytes(2, 'big') - + packet, 0x8864) - - -def llc_snap(packet: bytes) -> bytes: - """An 802.3 frame carrying ``packet`` behind LLC/SNAP.""" - body = b'\xaa\xaa\x03\x00\x00\x00' + (0x86DD if packet[0] >> 4 == 6 else 0x0800).to_bytes(2, 'big') + packet - return wire.MAC_B + wire.MAC_A + len(body).to_bytes(2, 'big') + body - - -def isl(frame: bytes) -> bytes: - """A Cisco ISL frame carrying the Ethernet ``frame``.""" - body = b'\xaa\xaa\x03\x00\x00\x0c\x00\x02\x00\x00\x00\x00' + frame - return b'\x01\x00\x0c\x00\x00\x00' + wire.MAC_A + len(body).to_bytes(2, 'big') + body + b'\x00' * 4 - - -def vlan(packet: bytes, *tpids: int) -> bytes: - """An Ethernet frame carrying ``packet`` behind a VLAN tag of each TPID in ``tpids``.""" - ethertype = 0x86DD if packet[0] >> 4 == 6 else 0x0800 - tags = b''.join(struct.pack('>HH', 1, tpid) for tpid in tpids[1:]) + struct.pack('>HH', 1, ethertype) - return wire.ethernet(tags + packet, tpids[0]) - - -def dstopts(count: int, nxt: int = 6) -> bytes: - """``count`` Destination Options headers in a row, the last naming ``nxt``.""" - return b''.join(bytes([60 if index < count - 1 else nxt, 0, 1, 4, 0, 0, 0, 0]) for index in range(count)) - - -def sll(packet: bytes) -> bytes: - """A Linux cooked-mode (SLL) frame carrying ``packet``.""" - return struct.pack('>HHH8sH', 0, 1, 6, wire.MAC_A + b'\x00\x00', 0x86DD if packet[0] >> 4 == 6 else 0x0800) + packet - - -def sll2(packet: bytes) -> bytes: - """A Linux cooked-mode v2 (SLL2) frame carrying ``packet``.""" - return struct.pack('>HHIHBB8s', 0x86DD if packet[0] >> 4 == 6 else 0x0800, 0, 1, 1, 0, 6, - wire.MAC_A + b'\x00\x00') + packet - - -def loopback(packet: bytes, family: int, byteorder: str) -> bytes: - """A BSD loopback frame carrying ``packet`` as the address ``family``, written in ``byteorder``.""" - return family.to_bytes(4, byteorder) + packet - - -#: An IPv4 Timestamp option of length 2, which the default engine's IPv4 parser rejects. -BAD_TIMESTAMP = b'\x44\x02\x00\x00' - - -def route(kind: int, pointer: int, *hops: bytes) -> bytes: - """An IPv4 source or record route option of ``kind`` through ``hops``, padded by End of Option List.""" - option = bytes([kind, 3 + 4 * len(hops), pointer]) + b''.join(hops) - return option + bytes(-len(option) % 4) - - -#: Two addresses to route through. -HOP_1, HOP_2 = bytes([10, 0, 0, 1]), bytes([10, 0, 0, 2]) - -#: An LSRR option through :data:`HOP_1` that no hop has used yet. -LSRR = route(0x83, 4, HOP_1) - -#: Ethernet frames carrying TCP, each with whether the default engine traces it, -#: by a name for it. Each builder takes the segment's number, its frame number. -#: The rows marked #1639 are the issue's; tshark traced every one of them. -ENCAPSULATION_CASES = { - 'control': (True, lambda n: eth(v4(tcp(n)))), - 'control, ipv6': (True, lambda n: eth(v6(tcp(n)))), - 'vlan': (True, lambda n: vlan(v4(tcp(n)), 0x8100)), - 'qinq, ipv6': (True, lambda n: vlan(v6(tcp(n)), 0x88A8, 0x8100)), - '4in4': (True, lambda n: eth(v4(v4(tcp(n), inner=True), 4))), - '6in4': (True, lambda n: eth(v4(v6(tcp(n), inner=True), 41))), - # #1639 - 'gre': (False, lambda n: eth(v4(gre(v4(tcp(n), inner=True)), 47))), - 'vxlan': (False, lambda n: eth(v4(vxlan(eth(v4(tcp(n), inner=True))), 17))), - 'mpls': (False, lambda n: eth(mpls(v4(tcp(n))), 0x8847)), - 'pppoe': (False, lambda n: pppoe(v4(tcp(n)))), - 'l2tp': (False, lambda n: eth(v4(l2tp(v4(tcp(n), inner=True)), 17))), - 'llc/snap': (False, lambda n: llc_snap(v4(tcp(n)))), - 'isl': (False, lambda n: isl(eth(v4(tcp(n))))), - # and around it - 'gre, ipv6': (False, lambda n: eth(v6(gre(v6(tcp(n), inner=True)), 47))), - 'gre, ethernet': (False, lambda n: eth(v4(gre(eth(v4(tcp(n), inner=True)), 0x6558), 47))), - 'gre in 4in4': (False, lambda n: eth(v4(v4(gre(v4(tcp(n))), 47, inner=True), 4))), - 'vxlan, ipv6': (False, lambda n: eth(v6(vxlan(eth(v6(tcp(n), inner=True))), 17))), - 'mpls, three labels, ipv6': (False, lambda n: eth(mpls(v6(tcp(n)), 3), 0x8847)), - 'vlan, mpls': (False, lambda n: wire.ethernet(struct.pack('>HH', 1, 0x8847) + mpls(v4(tcp(n))), 0x8100)), - 'pppoe, ipv6': (False, lambda n: pppoe(v6(tcp(n)))), - 'llc/snap, ipv6': (False, lambda n: llc_snap(v6(tcp(n)))), - 'isl, ipv6': (False, lambda n: isl(eth(v6(tcp(n))))), - 'teredo': (False, lambda n: eth(v4(wire.udp(v6(tcp(n)), dport=3544), 17))), - 'ipv4 ethertype, ipv6 packet': (False, lambda n: eth(v6(tcp(n)), 0x0800)), - # what the default engine reads of the IP layers themselves - 'rejected ipv4 options': (False, lambda n: eth(v4(tcp(n), opts=BAD_TIMESTAMP))), - 'rejected ipv4 options, inner': (False, lambda n: eth(v4(v4(tcp(n), inner=True, opts=BAD_TIMESTAMP), 4))), - '32 destination options': (True, lambda n: eth(v6(dstopts(32) + tcp(n), 60))), - '33 destination options': (False, lambda n: eth(v6(dstopts(33) + tcp(n), 60))), -} - -#: Frames of each link type carrying TCP, each with the link type and whether -#: the default engine traces it, by a name for it. -LINK_CASES = { - 'bsd raw (12)': (12, False, lambda n: v4(tcp(n))), - 'bsd raw (12), ipv6': (12, False, lambda n: v6(tcp(n))), - 'openbsd raw (14)': (14, False, lambda n: v4(tcp(n))), - 'openbsd raw (14), ipv6': (14, False, lambda n: v6(tcp(n))), - 'raw': (101, True, lambda n: v4(tcp(n))), - 'raw, ipv6': (101, True, lambda n: v6(tcp(n))), - 'raw, gre': (101, False, lambda n: v4(gre(v4(tcp(n), inner=True)), 47)), - 'ipv4': (228, True, lambda n: v4(tcp(n))), - # tshark reads an IPv6 packet here as IPv6, the default engine as IPv4 - 'ipv4, ipv6 packet': (228, False, lambda n: v6(tcp(n))), - 'ipv6': (229, True, lambda n: v6(tcp(n))), - 'ipv6, vxlan': (229, False, lambda n: v6(vxlan(eth(v4(tcp(n)))), 17)), - 'sll': (113, False, lambda n: sll(v4(tcp(n)))), - 'sll, ipv6': (113, False, lambda n: sll(v6(tcp(n)))), - 'sll2': (276, False, lambda n: sll2(v4(tcp(n)))), - 'sll2, ipv6': (276, False, lambda n: sll2(v6(tcp(n)))), - 'null': (0, True, lambda n: loopback(v4(tcp(n)), 2, 'little')), - 'null, ipv6': (0, True, lambda n: loopback(v6(tcp(n)), 30, 'big')), - 'null, gre': (0, False, lambda n: loopback(v4(gre(v4(tcp(n), inner=True)), 47), 2, 'little')), - 'loop': (108, True, lambda n: loopback(v4(tcp(n)), 2, 'big')), - 'loop, ipv6': (108, True, lambda n: loopback(v6(tcp(n)), 24, 'big')), - 'loop, gre, ipv6': (108, False, lambda n: loopback(v4(gre(v6(tcp(n), inner=True)), 47), 2, 'big')), - 'raw, lsrr': (101, True, lambda n: v4(tcp(n), opts=LSRR)), - 'null, lsrr, 4in4': (0, True, lambda n: loopback(v4(v4(tcp(n), inner=True), 4, opts=LSRR), 2, 'little')), -} - -#: Ethernet frames carrying TCP behind an IPv4 source or record route, as -#: :data:`ENCAPSULATION_CASES`. Where the route is not used up, tshark reports -#: its last address as ``ip.dst``; the default engine keys the segment by the -#: header's Destination Address. -SOURCE_ROUTE_CASES = { - 'lsrr, 4in4': (True, lambda n: eth(v4(v4(tcp(n), inner=True), 4, opts=LSRR))), - 'ssrr, 4in4': (True, lambda n: eth(v4(v4(tcp(n), inner=True), 4, opts=route(0x89, 4, HOP_1)))), - 'lsrr': (True, lambda n: eth(v4(tcp(n), opts=LSRR))), - 'ssrr': (True, lambda n: eth(v4(tcp(n), opts=route(0x89, 4, HOP_1)))), - 'lsrr, two hops': (True, lambda n: eth(v4(tcp(n), opts=route(0x83, 4, HOP_1, HOP_2)))), - 'lsrr, two hops, one used': (True, lambda n: eth(v4(tcp(n), opts=route(0x83, 8, HOP_1, HOP_2)))), - 'lsrr, used up': (True, lambda n: eth(v4(tcp(n), opts=route(0x83, 8, HOP_1)))), - 'record route': (True, lambda n: eth(v4(tcp(n), opts=route(0x07, 8, HOP_1)))), - 'lsrr, inner': (True, lambda n: eth(v4(v4(tcp(n), inner=True, opts=LSRR), 4))), - 'lsrr, 6in4': (True, lambda n: eth(v4(v6(tcp(n), inner=True), 41, opts=LSRR))), - 'lsrr, gre': (False, lambda n: eth(v4(gre(v4(tcp(n), inner=True)), 47, opts=LSRR))), -} - - -def build(cases: 'dict[str, tuple[bool, Callable[[int], bytes]]]') -> 'tuple[list[bytes], list[int]]': - """The frames of ``cases``, and the numbers of those the default engine traces TCP from.""" - numbered = list(enumerate(cases.values(), start=1)) - return ([make(number) for number, (_, make) in numbered], - [number for number, (reads, _) in numbered if reads]) - - -def block(kind: int, body: bytes, order: str = '<') -> bytes: - """A PCAP-NG block of type ``kind`` around ``body``, zero-padded to 32 bits, in byte order ``order``.""" - body += bytes(-len(body) % 4) - return struct.pack(f'{order}II', kind, len(body) + 12) + body + struct.pack(f'{order}I', len(body) + 12) - - -def section(linktypes: 'list[int]', packets: 'list[tuple[int, int, bytes]]', order: str = '<') -> bytes: - """A PCAP-NG section with an interface of each of ``linktypes``, then ``packets``. - - Each packet is ``(kind, interface, octets)``: an Enhanced Packet Block (6), - an obsolete Packet Block (2), a Simple Packet Block (3), which names no - interface, or a systemd Journal Export Block (9), which is no packet. - - """ - out = [block(0x0A0D0D0A, struct.pack(f'{order}IHHq', 0x1A2B3C4D, 1, 0, -1), order)] - out += [block(1, struct.pack(f'{order}HHI', linktype, 0, 0xFFFF), order) for linktype in linktypes] - for kind, interface, octets in packets: - head = {6: struct.pack(f'{order}IIIII', interface, 0, 1, len(octets), len(octets)), - 2: struct.pack(f'{order}HHIIII', interface, 0, 0, 1, len(octets), len(octets)), - 3: struct.pack(f'{order}I', len(octets))}.get(kind, b'') - out.append(block(kind, head + octets, order)) - return b''.join(out) - - -def read(test: unittest.TestCase, engine: str, capture: bytes, suffix: str = 'pcap') -> 'dict[str, Any]': - """Read ``capture`` with ``engine``, with TCP flow tracing on. - - Returns: - The input flow tracing receives for each frame number -- every field but - ``frame``, ``header`` and ``payload``, which PyShark cannot supply -- - and the traced flows' labels and frames. - - Skips ``test`` unless ``engine`` ran. - - """ - from pcapkit import extract - from pcapkit.foundation.traceflow.traceflow import TraceFlowBase - - if engine != 'default' and importlib.util.find_spec(engine) is None: - test.skipTest(f'{engine} not installed') - trace = TraceFlowBase.__call__ - seen = {'trace': {}} # type: dict[str, Any] - - def spy(this: 'Any', packet: 'Any') -> 'Any': - seen['trace'][packet.index] = {key: value for key, value in packet.items() - if key not in ('frame', 'header', 'payload')} - return trace(this, packet) - - with tempfile.TemporaryDirectory() as tmp, warnings.catch_warnings(), \ - mock.patch.object(TraceFlowBase, '__call__', spy): - warnings.simplefilter('ignore') - path = os.path.join(tmp, f'capture.{suffix}') - with open(path, 'wb') as file: - file.write(capture) - extractor = extract(fin=path, nofile=True, engine=engine, tcp=True, trace=True, - trace_fout=os.path.join(tmp, 'trace'), trace_format='json') - try: - if extractor._exnam != engine: # pylint: disable=protected-access - test.skipTest(f'{engine} did not run') - seen['flows'] = [(flow.label, tuple(flow.index)) for flow in extractor.trace.tcp] - finally: - close_extractor(extractor) - return seen - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class PySharkEncapsulationTraceTests(unittest.TestCase): - """The pyshark engine traces the TCP the default engine reads, and none inside what it leaves raw.""" - - def setUp(self) -> None: - reimport_once_per_class(self) - - def agree(self, capture: bytes, reads: 'list[int]', names: 'dict[int, str]', - suffix: str = 'pcap') -> 'dict[str, Any]': - base = read(self, 'default', capture, suffix) - # what the default engine reads, as the cases say - self.assertEqual([names[number] for number in sorted(base['trace'])], [names[number] for number in reads]) - seen = read(self, 'pyshark', capture, suffix) - self.assertEqual([names[number] for number in sorted(set(seen['trace']) ^ set(base['trace']))], [], - 'frames traced by one of pyshark and the default engine alone') - for number, record in base['trace'].items(): - self.assertEqual(seen['trace'][number], record, names[number]) - self.assertEqual(seen['flows'], base['flows']) - return seen - - def test_tcp_inside_an_encapsulation_the_default_engine_leaves_raw_is_not_traced(self) -> None: - frames, reads = build(ENCAPSULATION_CASES) - self.agree(wire.pcap([(1, number, octets) for number, octets in enumerate(frames)]), - reads, dict(enumerate(ENCAPSULATION_CASES, start=1))) - - def test_tcp_behind_a_source_route_is_keyed_by_the_header_destination(self) -> None: - # tshark reports the last address of a route not used up as ip.dst - frames, reads = build(SOURCE_ROUTE_CASES) - names = dict(enumerate(SOURCE_ROUTE_CASES, start=1)) - seen = self.agree(wire.pcap([(1, number, octets) for number, octets in enumerate(frames)]), reads, names) - number = {name: number for number, name in names.items()} - self.assertEqual(seen['trace'][number['lsrr']]['dst'], ipaddress.ip_address(wire.IPV4_B)) - self.assertEqual(seen['trace'][number['lsrr, 4in4']]['dst'], ipaddress.ip_address(wire.IPV4_A)) - - def test_each_link_type_is_read_as_the_default_engine_reads_it(self) -> None: - for linktype in sorted({link for link, _, _ in LINK_CASES.values()}): - cases = {name: (reads, make) for name, (link, reads, make) in LINK_CASES.items() if link == linktype} - with self.subTest(linktype=linktype): - frames, reads = build(cases) - self.agree(wire.pcap([(1, number, octets) for number, octets in enumerate(frames)], - linktype=linktype), reads, dict(enumerate(cases, start=1))) - - def test_each_pcapng_interface_is_read_by_its_own_link_type(self) -> None: - # tshark reads 12 and 14 as raw IP (101), and numbers a journal block - # as a frame where the default engine counts packets only - def raw(n: int) -> bytes: - return v4(tcp(n)) - - sections = [ - ([1, 12, 101, 113], '<', [(6, 0, eth(raw(1)), True), - (6, 1, raw(2), False), - (9, 0, b'__REALTIME_TIMESTAMP=1000000\nMESSAGE=hello\n', None), - (6, 2, raw(3), True), - (2, 3, sll(raw(4)), False), - (3, 0, eth(v4(gre(raw(5)), 47)), False), - (6, 0, eth(raw(6)), True)]), - ([14, 229], '>', [(3, 0, raw(7), False), - (6, 1, v6(tcp(8)), True), - (2, 0, v6(tcp(9)), False), - (6, 1, v6(vxlan(eth(raw(10))), 17), False), - (6, 1, v6(tcp(11)), True)]), - ] - capture = b''.join(section(linktypes, [packet[:3] for packet in packets], order) - for linktypes, order, packets in sections) - packets = [(f'section {index}, block {kind}, interface {interface}', reads) - for index, (_, _, blocks) in enumerate(sections, start=1) - for kind, interface, _, reads in blocks if reads is not None] - self.agree(capture, [number for number, (_, reads) in enumerate(packets, start=1) if reads], - {number: f'{number}: {name}' for number, (name, _) in enumerate(packets, start=1)}, - suffix='pcapng') - - def test_frames_the_second_pass_cannot_read_are_traced_as_tshark_dissects_them(self) -> None: - # as before #1639, with one warning - from pcapkit.utilities.warnings import PySharkWarning - - cases = {name: ENCAPSULATION_CASES[name] for name in ('control', 'gre', 'mpls', 'control, ipv6')} - frames, _ = build(cases) - capture = wire.pcap([(1, number, octets) for number, octets in enumerate(frames)]) - with mock.patch('pcapkit.toolkit.pyshark.OCTETS_FILTER', 'tcp ||'), \ - mock.patch('pcapkit.foundation.engines.pyshark.warn') as warn: - seen = read(self, 'pyshark', capture) - self.assertEqual(sorted(seen['trace']), [1, 2, 3, 4]) - messages = [call.args[0] for call in warn.call_args_list if call.args[1] is PySharkWarning] - self.assertEqual(len(messages), 1, messages) - self.assertIn('Frame 1: the second tshark pass, which reads the captured octets of a frame with an AH ' - 'or IPv6 extension header in IPv4, with TCP, or with more 802.1Q tags than tshark dissects,', - messages[0]) - - -class FakeField(str): - """A PyShark field: text, with the position and raw value tshark reports for it.""" - - def __new__(cls, show: str, pos: int = 0, raw_value: str = '') -> 'FakeField': - field = super().__new__(cls, show) - field.pos, field.raw_value = str(pos), raw_value - return field - - -class FakeLayer: - def __init__(self, layer_name: str, **fields: object) -> None: - self.layer_name = layer_name - self.field_names = tuple(fields) - for name, value in fields.items(): - setattr(self, name, value) - - -def ip_layer(octets: bytes, start: int) -> FakeLayer: - """The ``ip`` or ``ipv6`` layer tshark reports for the IP header at ``start`` of ``octets``.""" - if octets[start] >> 4 == 4: - src, dst = octets[start + 12:start + 16], octets[start + 16:start + 20] - return FakeLayer('ip', version=FakeField('4', start, '4'), proto=FakeField(str(octets[start + 9])), - src=FakeField(str(ipaddress.ip_address(src)), start + 12, src.hex()), - dst=FakeField(str(ipaddress.ip_address(dst)), start + 16, dst.hex())) - src, dst = octets[start + 8:start + 24], octets[start + 24:start + 40] - return FakeLayer('ipv6', version=FakeField('6', start, '6'), nxt=FakeField(str(octets[start + 6])), - src=FakeField(str(ipaddress.ip_address(src)), start + 8, src.hex()), - dst=FakeField(str(ipaddress.ip_address(dst)), start + 24, dst.hex())) - - -def tcp_layer(octets: bytes, start: int, **fields: str) -> FakeLayer: - """The ``tcp`` layer tshark reports for the TCP header at ``start`` of ``octets``, with ``fields`` replaced.""" - flags = octets[start + 13] - names = {'srcport': str(int.from_bytes(octets[start:start + 2], 'big')), - 'dstport': str(int.from_bytes(octets[start + 2:start + 4], 'big')), - 'seq_raw': str(int.from_bytes(octets[start + 4:start + 8], 'big')), - 'ack_raw': str(int.from_bytes(octets[start + 8:start + 12], 'big')), - 'flags_syn': str(bool(flags & 0x02)), 'flags_fin': str(bool(flags & 0x01)), - 'flags_reset': str(bool(flags & 0x04)), 'urgent_pointer': '0'} - names.update(fields) - return FakeLayer('tcp', **names) - - -def fake_packet(link: str, encap: str, layers: 'list[FakeLayer]', number: str = '3') -> 'Any': - """A PyShark packet: its frame layer of ``encap``, a ``link`` layer, then ``layers``.""" - frame = FakeLayer('frame', encap_type=encap, time_epoch='1.5') - return types.SimpleNamespace(number=number, frame_info=frame, sniff_timestamp='1.5', - layers=[frame, FakeLayer(link), *layers]) - - -#: An Ethernet frame of IPv4 and TCP, and tshark's dissection of it. -PLAIN = eth(v4(tcp(7))) - - -def plain(**fields: str) -> 'Any': - """tshark's dissection of :data:`PLAIN`, with ``fields`` of its TCP layer replaced.""" - return fake_packet('eth', '1', [ip_layer(PLAIN, 14), tcp_layer(PLAIN, 34, **fields)]) - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class ToolkitEncapsulationTests(unittest.TestCase): - """:func:`tcp_traceflow` reads a frame tshark finds TCP in from its octets, with no tshark.""" - - def setUp(self) -> None: - reimport_once_per_class(self) - - def test_a_frame_tshark_traces_tcp_in_is_read_from_its_octets(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - # tshark's fields are not read: its ports here are not the frame's - asked = [] # type: list[Any] - packet = plain(srcport='1', dstport='2') - data = toolkit.tcp_traceflow(packet, count=4, - captured=lambda p: asked.append(p) or toolkit.Captured(1, PLAIN)) - self.assertEqual(asked, [packet]) - self.assertEqual((data.index, data.src, data.dst, data.srcport, data.dstport, data.seq, data.ack), - (4, ipaddress.ip_address('192.0.2.1'), ipaddress.ip_address('198.51.100.2'), - 1007, 9, 1007, 1)) - # nor are its flags, so a spelling it does not use raises nothing here - self.assertIsNotNone(toolkit.tcp_traceflow(plain(flags_syn='maybe'), - captured=lambda p: toolkit.Captured(1, PLAIN))) - - # where the frame cannot be had, tshark's dissection is traced, as before - for captured in (None, lambda p: None): - with self.subTest(captured=captured): - self.assertEqual(toolkit.tcp_traceflow(plain(srcport='1'), captured=captured).srcport, 1) - - # and a frame tshark finds no TCP in is not asked for - def refuse(packet: 'Any') -> 'Any': - raise AssertionError('octets asked for') - - for layers in ([ip_layer(PLAIN, 14)], [ip_layer(PLAIN, 14), FakeLayer('tcp', srcport='1')], - [tcp_layer(PLAIN, 34), ip_layer(PLAIN, 14)]): - with self.subTest(layers=[layer.layer_name for layer in layers]): - self.assertIsNone(toolkit.tcp_traceflow(fake_packet('eth', '1', layers), captured=refuse)) - - def test_tcp_over_a_link_type_the_default_engine_reads_no_ip_over_is_not_traced(self) -> None: - # tshark reads 12 and 14 as raw IP (frame.encap_type 7, as for 101) - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit import pyshark as toolkit - - octets = v4(tcp(7)) - packet = fake_packet('raw', '7', [ip_layer(octets, 0), tcp_layer(octets, 20)]) - for linktype in (12, 14, 113, 276, 9, 999): - with self.subTest(linktype=linktype): - self.assertIsNone(toolkit.tcp_traceflow( - packet, captured=lambda p, l=linktype: toolkit.Captured(l, octets))) - data = toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(101, octets)) - self.assertEqual((data.protocol, data.srcport), (LinkType.RAW, 1007)) - - # and nothing is raised for a link type of tshark's this table lacks, - # as no segment is found - packet.frame_info.encap_type = '999' - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(113, sll(octets)))) - - def test_tcp_inside_an_encapsulation_the_default_engine_leaves_raw_is_not_traced(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - inner = v4(tcp(7), inner=True) - for name, octets, link, layers in ( - ('gre', eth(v4(gre(inner), 47)), 'eth', lambda o: [ip_layer(o, 14), FakeLayer('gre'), ip_layer(o, 38), - tcp_layer(o, 58)]), - ('vxlan', eth(v4(vxlan(eth(inner)), 17)), 'eth', - lambda o: [ip_layer(o, 14), FakeLayer('udp'), FakeLayer('vxlan'), FakeLayer('eth'), ip_layer(o, 64), - tcp_layer(o, 84)]), - ('mpls', eth(mpls(inner), 0x8847), 'eth', lambda o: [FakeLayer('mpls'), ip_layer(o, 18), tcp_layer(o, 38)]), - ('llc/snap', llc_snap(inner), 'eth', lambda o: [FakeLayer('llc'), ip_layer(o, 22), tcp_layer(o, 42)]), - ('isl', isl(eth(inner)), 'isl', lambda o: [FakeLayer('eth'), ip_layer(o, 40), tcp_layer(o, 60)]), - ): - with self.subTest(case=name): - packet = fake_packet(link, '1', layers(octets)) - self.assertIsNotNone(toolkit.tcp_traceflow(packet), 'tshark traces it') - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p, o=octets: toolkit.Captured(1, o))) - # IP-in-IP is a tunnel the default engine dissects - octets = eth(v4(inner, 4)) - packet = fake_packet('eth', '1', [ip_layer(octets, 14), ip_layer(octets, 34), tcp_layer(octets, 54)]) - data = toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(1, octets)) - self.assertEqual(data.src, ipaddress.ip_address('198.51.100.2')) - - def test_tcp_whose_ip_layer_tshark_dissects_elsewhere_is_not_traced(self) -> None: - # over link type IPV4 (228), tshark reads an IPv6 packet as IPv6, its - # ``ip`` layer the version alone; the default engine reads it as IPv4 - from pcapkit.toolkit import pyshark as toolkit - - octets = v6(tcp(7)) - packet = fake_packet('raw', '129', [FakeLayer('ip', version=FakeField('6', 0, '6')), ip_layer(octets, 0), - tcp_layer(octets, 40)]) - self.assertIsNotNone(toolkit.tcp_traceflow(packet), 'tshark traces it') - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(228, octets))) - # and over IPV6 (229), where both read it as IPv6, it is traced - packet = fake_packet('raw', '130', [ip_layer(octets, 0), tcp_layer(octets, 40)]) - self.assertEqual(toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(229, octets)).srcport, 1007) - - def test_a_source_routed_header_is_read_from_its_octets(self) -> None: - # tshark reports the last address of a route not used up as ip.dst, at - # its place in the option, and the header's as ip.cur_rt - from pcapkit.toolkit import pyshark as toolkit - - def routed(octets: bytes, start: int, pos: int, raw: bytes, **fields: FakeField) -> FakeLayer: - """:func:`ip_layer`'s, with the header's destination as ``cur_rt``, ``raw`` at ``pos`` as ``dst``.""" - layer = ip_layer(octets, start) - return FakeLayer('ip', **{**{name: getattr(layer, name) for name in layer.field_names}, - 'cur_rt': layer.dst, - 'dst': FakeField(str(ipaddress.ip_address(raw)), pos, raw.hex()), **fields}) - - plain_ = eth(v4(tcp(7), opts=LSRR)) - tunnel = eth(v4(v4(tcp(7), inner=True), 4, opts=LSRR)) - for name, octets, layers, dst in ( - ('lsrr', plain_, lambda o: [routed(o, 14, 37, HOP_1), tcp_layer(o, 42)], wire.IPV4_B), - ('lsrr, 4in4', tunnel, lambda o: [routed(o, 14, 37, HOP_1), ip_layer(o, 42), tcp_layer(o, 62)], - wire.IPV4_A), - ): - with self.subTest(case=name): - packet = fake_packet('eth', '1', layers(octets)) - data = toolkit.tcp_traceflow(packet, captured=lambda p, o=octets: toolkit.Captured(1, o)) - self.assertIsNotNone(data) - self.assertEqual((data.dst, data.srcport), (ipaddress.ip_address(dst), 1007)) - - # but each address must still be in the octets where tshark reports it - for name, layer in ( - ('another destination address', routed(plain_, 14, 37, HOP_2)), - ('the destination address before the header', routed(plain_, 14, 0, plain_[0:4])), - ('another source address', routed(plain_, 14, 37, HOP_1, src=FakeField('10.0.0.2', 26, HOP_2.hex()))), - ): - with self.subTest(case=name): - packet = fake_packet('eth', '1', [layer, tcp_layer(plain_, 42)]) - self.assertIsNotNone(toolkit.tcp_traceflow(packet), 'tshark traces it') - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(1, plain_))) - - def test_the_filter_matches_every_frame_tshark_traces_tcp_in(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - self.assertEqual(toolkit.OCTETS_FILTER.split(' || ', maxsplit=1)[0], 'tcp') - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/toolkit/test_pyshark_fallback_and_layer_bound_1609_1671_unit.py b/tests/toolkit/test_pyshark_fallback_and_layer_bound_1609_1671_unit.py deleted file mode 100644 index edf73e2a1..000000000 --- a/tests/toolkit/test_pyshark_fallback_and_layer_bound_1609_1671_unit.py +++ /dev/null @@ -1,327 +0,0 @@ -# -*- coding: utf-8 -*- -"""pyshark keys a segment by its header's addresses, and reads within the default engine's bounds. C.f. #1609, #1671. - -Where a frame's octets cannot be had, pyshark traces the TCP tshark dissects. -It took tshark's ``ip.dst`` for the destination there. tshark reports none -behind an LSRR or SSRR option whose route data is no whole number of -addresses, so a bare :exc:`AttributeError` ended the extraction; and behind a -route not yet used up, ``ip.dst`` is the route's last address, so the segment -was keyed by that hop. It now takes the header's destination, tshark's -``ip.cur_rt`` there, and reads no segment where tshark reports no destination, -as the default engine's parser rejects such an option (#1671). - -Read from its octets, a frame was followed through IP-in-IP, AH and extension -headers past its 64th layer, where the default engine dissects nothing. And a -frame tshark stops in was not read at all: tshark dissects no more than 20 -802.1Q tags of a frame. pyshark now counts the layers as the default engine -does, and reads a frame tshark stops in a stack of 802.1Q tags from its octets -too (#1609). - -Every capture is built here, so the module is in the unit tier. The tests that -run tshark skip unless the pyshark engine runs, which needs ``pyshark``, the -``tshark`` binary and Python 3.13 or older; the rest use fakes. - -""" -from __future__ import annotations - -import ipaddress -import struct -import unittest -from typing import TYPE_CHECKING -from unittest import mock - -from tests._support import reimport_once_per_class -from tests.foundation import _roundtrip as wire -from tests.toolkit.test_pyshark_encapsulated_tcp_1639_unit import (BAD_TIMESTAMP, HAS_RUNTIME, - HOP_1, HOP_2, LSRR, FakeField, - FakeLayer, build, eth, - fake_packet, ip_layer, read, - route, tcp, tcp_layer, v4, v6, - vlan) -from tests.toolkit.test_tcp_carrier_walk_1606_1617_unit import ah, options - -if TYPE_CHECKING: - from typing import Any, Callable - -#: An 802.1Q and an 802.1ad tag's TPID. -C_TAG, S_TAG = 0x8100, 0x88A8 - - -def option(kind: int, length: int, pointer: int, data: bytes) -> bytes: - """An IPv4 option of ``kind`` claiming ``length`` octets, then ``pointer`` and ``data``, padded to 32 bits.""" - raw = bytes([kind, length, pointer]) + data - return raw + bytes(-len(raw) % 4) - - -#: An LSRR option of length 9, whose route data is no whole number of addresses. -LSRR_9 = option(0x83, 9, 4, HOP_1 + b'\x01\x02') - - -def nest(packet: bytes, count: int, outer: int = 4) -> bytes: - """``packet`` tunnelled in ``count`` IPv4 (``outer`` 4) or IPv6 (6) headers.""" - for _ in range(count): - proto = 41 if packet[0] >> 4 == 6 else 4 - packet = v4(packet, proto) if outer == 4 else v6(packet, proto) - return packet - - -def pairs(count: int, ipv6: bool, number: int) -> bytes: - """TCP behind ``count`` IPv4 packets, each with an AH, or IPv6 packets, each with a Hop-by-Hop header.""" - packet, nxt = tcp(number), 6 - for _ in range(count): - packet = v6(options(nxt) + packet, 0) if ipv6 else v4(ah(nxt) + packet, 51) - nxt = 41 if ipv6 else 4 - return packet - - -def chain(count: int, nxt: int = 6) -> bytes: - """``count`` Destination Options headers in a row, the last naming ``nxt``.""" - return b''.join(bytes([60 if index < count - 1 else nxt, 0, 1, 4, 0, 0, 0, 0]) for index in range(count)) - - -#: Ethernet frames carrying TCP behind an IPv4 route option, each with whether -#: the default engine traces it, by a name for it, as -#: :data:`~tests.toolkit.test_pyshark_encapsulated_tcp_1639_unit.ENCAPSULATION_CASES`. -#: tshark reports no ``ip.dst`` behind each malformed route, and the last hop -#: as ``ip.dst`` behind each route not used up. -ROUTE_CASES = { - 'control': (True, lambda n: eth(v4(tcp(n)))), - 'lsrr of length 9': (False, lambda n: eth(v4(tcp(n), opts=LSRR_9))), - 'ssrr of length 9': (False, lambda n: eth(v4(tcp(n), opts=option(0x89, 9, 4, HOP_1 + b'\x01\x02')))), - 'lsrr of length 9, one hop used': (False, lambda n: eth(v4(tcp(n), opts=option(0x83, 9, 8, HOP_1 + b'\x01\x02')))), - 'lsrr of length 5': (False, lambda n: eth(v4(tcp(n), opts=option(0x83, 5, 4, b'\x01\x02')))), - 'lsrr of length 13': (False, lambda n: eth(v4(tcp(n), opts=option(0x83, 13, 4, HOP_1 + bytes(6))))), - 'lsrr of length 9, inner': (False, lambda n: eth(v4(v4(tcp(n), inner=True, opts=LSRR_9), 4))), - 'lsrr': (True, lambda n: eth(v4(tcp(n), opts=LSRR))), - 'ssrr': (True, lambda n: eth(v4(tcp(n), opts=route(0x89, 4, HOP_1)))), - 'lsrr, two hops, one used': (True, lambda n: eth(v4(tcp(n), opts=route(0x83, 8, HOP_1, HOP_2)))), - 'lsrr, used up': (True, lambda n: eth(v4(tcp(n), opts=route(0x83, 8, HOP_1)))), - 'record route': (True, lambda n: eth(v4(tcp(n), opts=route(0x07, 8, HOP_1)))), - 'lsrr, inner': (True, lambda n: eth(v4(v4(tcp(n), inner=True, opts=LSRR), 4))), - 'lsrr, 4in4': (True, lambda n: eth(v4(v4(tcp(n), inner=True), 4, opts=LSRR))), - 'lsrr, 6in4': (True, lambda n: eth(v4(v6(tcp(n), inner=True), 41, opts=LSRR))), - 'control, ipv6': (True, lambda n: eth(v6(tcp(n)))), - 'control, after the rest': (True, lambda n: eth(v4(tcp(n)))), -} - -#: Ethernet frames carrying TCP around the 64-layer bound of a frame, Ethernet -#: being layer 1, and each VLAN tag, IP header, AH and IPv6 extension header one -#: more; by a name for each, with whether the default engine traces it. tshark -#: traces every one but those behind more than 20 802.1Q tags. -BOUND_CASES = { - 'tcp in 62 ipv4 (layer 64)': (True, lambda n: eth(nest(v4(tcp(n)), 61))), - 'tcp in 63 ipv4 (layer 65)': (False, lambda n: eth(nest(v4(tcp(n)), 62))), - 'tcp in 100 ipv4': (False, lambda n: eth(nest(v4(tcp(n)), 99))), - 'tcp in 62 ipv6 (layer 64)': (True, lambda n: eth(nest(v6(tcp(n)), 61, 6))), - 'tcp in 63 ipv6 (layer 65)': (False, lambda n: eth(nest(v6(tcp(n)), 62, 6))), - 'tcp in ipv4 in 61 ipv6 (layer 64)': (True, lambda n: eth(nest(v4(tcp(n)), 61, 6))), - 'tcp in ipv4 in 62 ipv6 (layer 65)': (False, lambda n: eth(nest(v4(tcp(n)), 62, 6))), - 'tcp behind 31 (ipv4, ah) pairs (layer 64)': (True, lambda n: eth(pairs(31, False, n))), - 'tcp behind 32 (ipv4, ah) pairs (layer 66)': (False, lambda n: eth(pairs(32, False, n))), - 'tcp behind 31 (ipv6, hop-by-hop) pairs (layer 64)': (True, lambda n: eth(pairs(31, True, n))), - 'tcp behind 32 (ipv6, hop-by-hop) pairs (layer 66)': (False, lambda n: eth(pairs(32, True, n))), - 'tcp behind 61 ipv4 and a hop-by-hop header (layer 64)': ( - True, lambda n: eth(nest(v4(options(6) + tcp(n), 0), 60))), - 'tcp behind 62 ipv4 and a hop-by-hop header (layer 65)': ( - False, lambda n: eth(nest(v4(options(6) + tcp(n), 0), 61))), - 'tcp behind 50 ipv4, ipv6 and 11 extension headers (layer 64)': ( - True, lambda n: eth(nest(v6(chain(11) + tcp(n), 60), 50))), - 'tcp behind 50 ipv4, ipv6 and 12 extension headers (layer 65)': ( - False, lambda n: eth(nest(v6(chain(12) + tcp(n), 60), 50))), - 'tcp behind 61 802.1ad tags (layer 64)': (True, lambda n: vlan(v4(tcp(n)), *[S_TAG] * 61)), - 'tcp behind 62 802.1ad tags (layer 65)': (False, lambda n: vlan(v4(tcp(n)), *[S_TAG] * 62)), - # tshark dissects at most 20 802.1Q tags of a frame - 'tcp behind 20 802.1q tags': (True, lambda n: vlan(v4(tcp(n)), *[C_TAG] * 20)), - 'tcp behind 21 802.1q tags': (True, lambda n: vlan(v4(tcp(n)), *[C_TAG] * 21)), - 'tcp behind 30 802.1q tags, ipv6': (True, lambda n: vlan(v6(tcp(n)), *[C_TAG] * 30)), - 'tcp behind 42 802.1ad and 802.1q tags': (True, lambda n: vlan(v4(tcp(n)), *[S_TAG, C_TAG] * 21)), - 'tcp in 4in4 behind 30 802.1q tags': (True, lambda n: vlan(v4(v4(tcp(n), inner=True), 4), *[C_TAG] * 30)), - 'tcp behind 61 802.1q tags (layer 64)': (True, lambda n: vlan(v4(tcp(n)), *[C_TAG] * 61)), - 'tcp behind 62 802.1q tags (layer 65)': (False, lambda n: vlan(v4(tcp(n)), *[C_TAG] * 62)), - 'udp behind 30 802.1q tags': (False, lambda n: vlan(v4(wire.udp(b'data'), 17), *[C_TAG] * 30)), - 'tcp behind 30 802.1q tags, rejected ipv4 options': ( - False, lambda n: vlan(v4(tcp(n), opts=BAD_TIMESTAMP), *[C_TAG] * 30)), -} - -#: Frames of other link types carrying TCP around the bound, by a name for -#: each, with the link type and whether the default engine traces it. -LINK_BOUND_CASES = { - 'null, tcp in 62 ipv4 (layer 64)': (0, True, lambda n: b'\x02\x00\x00\x00' + nest(v4(tcp(n)), 61)), - 'null, tcp in 63 ipv4 (layer 65)': (0, False, lambda n: b'\x02\x00\x00\x00' + nest(v4(tcp(n)), 62)), - 'raw, tcp in 63 ipv4 (layer 64)': (101, True, lambda n: nest(v4(tcp(n)), 62)), - 'raw, tcp in 64 ipv4 (layer 65)': (101, False, lambda n: nest(v4(tcp(n)), 63)), -} - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class PySharkFallbackAndBoundTests(unittest.TestCase): - """The pyshark engine traces the TCP the default engine reads, as the cases say.""" - - def setUp(self) -> None: - reimport_once_per_class(self) - - def agree(self, cases: 'dict[str, tuple[bool, Callable[[int], bytes]]]', linktype: int = 1) -> None: - frames, reads = build(cases) - capture = wire.pcap([(1, number, octets) for number, octets in enumerate(frames)], linktype=linktype) - names = dict(enumerate(cases, start=1)) - base = read(self, 'default', capture) - # what the default engine reads, as the cases say - self.assertEqual([names[number] for number in sorted(base['trace'])], [names[number] for number in reads]) - seen = read(self, 'pyshark', capture) - self.assertEqual({names[number]: seen['trace'].get(number) for number in names}, - {names[number]: base['trace'].get(number) for number in names}) - self.assertEqual(seen['flows'], base['flows']) - - def test_where_the_octets_cannot_be_had_a_route_option_ends_no_extraction(self) -> None: - # the second pass fails, so every frame is traced as tshark dissects - # it: the header's destination, and nothing where tshark reports none - with mock.patch('pcapkit.toolkit.pyshark.OCTETS_FILTER', 'tcp ||'): - self.agree(ROUTE_CASES) - self.agree({'raw, lsrr': (True, lambda n: v4(tcp(n), opts=LSRR)), - 'raw, lsrr of length 9': (False, lambda n: v4(tcp(n), opts=LSRR_9))}, 101) - - def test_nothing_is_traced_past_the_frame_layer_bound(self) -> None: - self.agree(BOUND_CASES) - for linktype in sorted({link for link, _, _ in LINK_BOUND_CASES.values()}): - with self.subTest(linktype=linktype): - self.agree({name: (reads, make) for name, (link, reads, make) in LINK_BOUND_CASES.items() - if link == linktype}, linktype) - - -def routed(octets: bytes, start: int, hop: bytes) -> FakeLayer: - """tshark's ``ip`` layer behind a route not used up: the header's destination as ``cur_rt``, ``hop`` as ``dst``.""" - layer = ip_layer(octets, start) - return FakeLayer('ip', **{**{name: getattr(layer, name) for name in layer.field_names}, 'cur_rt': layer.dst, - 'dst': FakeField(str(ipaddress.ip_address(hop)), start + 23, hop.hex())}) - - -def bare(layer: FakeLayer, *names: str) -> FakeLayer: - """``layer`` without its fields ``names``, as tshark reports a layer it does not finish.""" - return FakeLayer(layer.layer_name, **{name: getattr(layer, name) for name in layer.field_names - if name not in names}) - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class ToolkitFallbackTests(unittest.TestCase): - """Where the frame cannot be had, :func:`tcp_traceflow` keys tshark's segment by the header; no tshark.""" - - def setUp(self) -> None: - reimport_once_per_class(self) - - def test_a_header_tshark_reports_no_address_of_is_not_traced(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - octets = eth(v4(tcp(7), opts=LSRR_9)) - header = ip_layer(octets, 14) - for name, layer in (('no destination', bare(header, 'dst')), - ('no destination, the header\'s as cur_rt', FakeLayer('ip', **{ - **{field: getattr(header, field) for field in header.field_names if field != 'dst'}, - 'cur_rt': header.dst})), - ('no source', bare(header, 'src'))): - for captured in (None, lambda p: None): - with self.subTest(case=name, captured=captured): - packet = fake_packet('eth', '1', [layer, tcp_layer(octets, 46)]) - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=captured)) - - def test_a_source_routed_segment_is_keyed_by_the_header_destination(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - plain = eth(v4(tcp(7), opts=LSRR)) - tunnel = eth(v4(v4(tcp(7), inner=True, opts=LSRR), 4)) # the inner header's addresses swapped - for name, packet, addresses in ( - ('lsrr', fake_packet('eth', '1', [routed(plain, 14, HOP_1), tcp_layer(plain, 42)]), - (wire.IPV4_A, wire.IPV4_B)), - ('lsrr, inner', fake_packet('eth', '1', [ip_layer(tunnel, 14), routed(tunnel, 34, HOP_1), - tcp_layer(tunnel, 62)]), (wire.IPV4_B, wire.IPV4_A)), - ): - for captured in (None, lambda p: None): - with self.subTest(case=name, captured=captured): - data = toolkit.tcp_traceflow(packet, captured=captured) - self.assertEqual((data.src, data.dst, data.srcport), - (*map(ipaddress.ip_address, addresses), 1007)) - # an IPv6 header's destination is tshark's ipv6.dst - octets = eth(v6(tcp(7))) - data = toolkit.tcp_traceflow(fake_packet('eth', '1', [ip_layer(octets, 14), tcp_layer(octets, 54)])) - self.assertEqual(data.dst, ipaddress.ip_address(wire.IPV6_B)) - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class ToolkitBoundTests(unittest.TestCase): - """:func:`tcp_traceflow` reads a frame's octets within the default engine's bounds; no tshark.""" - - def setUp(self) -> None: - reimport_once_per_class(self) - - def trace(self, linktype: int, octets: bytes, ip: int, at: int, link: str = 'eth') -> 'Any': - """Trace ``octets`` as tshark dissects them: the IP header at ``ip``, the TCP one at ``at``.""" - from pcapkit.toolkit import pyshark as toolkit - - packet = fake_packet(link, '1', [ip_layer(octets, ip), tcp_layer(octets, at)]) - return toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(linktype, octets)) - - def test_no_tcp_is_read_past_the_64th_layer(self) -> None: - # Ethernet is layer 1, then each IP header, AH and extension header - reads = {} - for count, layer in ((61, 64), (62, 65)): - octets = eth(nest(v4(tcp(7)), count)) - reads[f'tcp in {count + 1} ipv4 (layer {layer})'] = self.trace(1, octets, 14, 14 + 20 * (count + 1)) - octets = nest(v4(tcp(7)), count + 1) - reads[f'raw, tcp in {count + 2} ipv4 (layer {layer})'] = self.trace(101, octets, 0, 20 * (count + 2), 'raw') - octets = eth(nest(v6(tcp(7)), count, 6)) - reads[f'tcp in {count + 1} ipv6 (layer {layer})'] = self.trace(1, octets, 14, 14 + 40 * (count + 1)) - for headers, layer in ((11, 64), (12, 65), (13, 66)): - octets = eth(nest(v6(chain(headers) + tcp(7), 60), 50)) - reads[f'tcp behind 50 ipv4, ipv6 and {headers} extension headers (layer {layer})'] = self.trace( - 1, octets, 14, 14 + 50 * 20 + 40 + 8 * headers) - for count, layer in ((31, 64), (32, 66)): - octets = eth(pairs(count, False, 7)) # each pair 44 octets: an IPv4 header and a 24-octet AH - reads[f'tcp behind {count} (ipv4, ah) pairs (layer {layer})'] = self.trace(1, octets, 14, - 14 + 44 * count) - self.assertEqual({name: data is not None for name, data in reads.items()}, { - 'tcp in 62 ipv4 (layer 64)': True, 'raw, tcp in 63 ipv4 (layer 64)': True, - 'tcp in 62 ipv6 (layer 64)': True, - 'tcp in 63 ipv4 (layer 65)': False, 'raw, tcp in 64 ipv4 (layer 65)': False, - 'tcp in 63 ipv6 (layer 65)': False, - 'tcp behind 50 ipv4, ipv6 and 11 extension headers (layer 64)': True, - 'tcp behind 50 ipv4, ipv6 and 12 extension headers (layer 65)': False, - 'tcp behind 50 ipv4, ipv6 and 13 extension headers (layer 66)': False, - 'tcp behind 31 (ipv4, ah) pairs (layer 64)': True, - 'tcp behind 32 (ipv4, ah) pairs (layer 66)': False, - }) - - def test_a_frame_tshark_stops_in_802_1q_tags_is_read_from_its_octets(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - # tshark dissects 21 vlan layers, the last with "Too many nested VLAN tags", and stops - stopped = [FakeLayer('vlan') for _ in range(21)] - reads = {} - for count, layer in ((30, 33), (61, 64), (62, 65)): - octets = vlan(v4(tcp(7)), *[C_TAG] * count) - asked = [] # type: list[Any] - packet = fake_packet('eth', '1', stopped) - data = toolkit.tcp_traceflow(packet, captured=lambda p, o=octets: asked.append(p) - or toolkit.Captured(1, o)) - self.assertEqual(asked, [packet]) - reads[f'tcp behind {count} 802.1q tags (layer {layer})'] = None if data is None else data.srcport - octets = wire.ethernet(struct.pack('>HH', 1, C_TAG) * 29 + struct.pack('>HH', 1, 0x0806) + bytes(28), C_TAG) - reads['arp behind 30 802.1q tags'] = toolkit.tcp_traceflow( - fake_packet('eth', '1', stopped), captured=lambda p: toolkit.Captured(1, octets)) - self.assertEqual(reads, {'tcp behind 30 802.1q tags (layer 33)': 1007, - 'tcp behind 61 802.1q tags (layer 64)': 1007, - 'tcp behind 62 802.1q tags (layer 65)': None, - 'arp behind 30 802.1q tags': None}) - - # a frame tshark dissects an IP layer behind its tags in is not asked - # for, unless it finds TCP there; and the second pass writes every - # frame with a VLAN tag - def refuse(packet: 'Any') -> 'Any': - raise AssertionError('octets asked for') - - octets = vlan(v4(wire.udp(b'data'), 17), C_TAG) - self.assertIsNone(toolkit.tcp_traceflow(fake_packet('eth', '1', [FakeLayer('vlan'), ip_layer(octets, 18), - FakeLayer('udp')]), captured=refuse)) - self.assertEqual(toolkit.OCTETS_FILTER.rsplit(' || ', maxsplit=1)[1], 'vlan') - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/toolkit/test_pyshark_ipv4_extension_headers_1625_unit.py b/tests/toolkit/test_pyshark_ipv4_extension_headers_1625_unit.py deleted file mode 100644 index d28cd1489..000000000 --- a/tests/toolkit/test_pyshark_ipv4_extension_headers_1625_unit.py +++ /dev/null @@ -1,1082 +0,0 @@ -# -*- coding: utf-8 -*- -"""The pyshark engine traces the TCP the default engine reads behind an extension header in IPv4. C.f. #1625. - -The default engine dispatches an IPv4 packet's protocol through the registry -IPv6 uses, so it reads an AH or IPv6 extension header carried directly in IPv4, -and the TCP segment or IP-in-IP tunnel behind it, up to -:data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` headers in a -row; behind such a header it reads TCP only if its TCP parser accepts the -header. tshark dissects TCP behind such a header by rules of its own, so the -pyshark engine traced TCP the default engine rejects, and none at all behind a -Mobility or HIP header, past which tshark dissects nothing and whose payload the -PDML PyShark reads leaves out. Nor does the default engine read past Shim6 in -IPv4, whose parser refuses it, where tshark does. For such a frame the engine -now reads the captured octets from a second tshark pass and applies the -default engine's rules to them -(:func:`pcapkit.toolkit.pypcapfile._past_extension_headers`), from the -outermost IP layer the default engine dissects on, so that a tunnel it does not -dissect, such as GRE or VXLAN, or the chain bound of an outer IPv6 header, -stops the read as it stops the default engine. That layer is found by the link -type in the capture file, which the engine reads for itself, as tshark reads -link types 12 and 14 as it reads raw IP (101). A frame of a link type or -EtherType the default engine dissects no IP over is not traced, as the -default engine traces none there (#1639). - -That pass is a process of the engine's own. If it fails, or the capture file -cannot be read for a link type, the engine warns and traces such frames as -tshark dissects them; whether it fails or not, closing the engine reaps it, -and the first pass's tshark with it. - -Every capture is built here, so the module is in the unit tier. The tests that -run tshark skip unless the pyshark engine runs, which needs ``pyshark``, the -``tshark`` binary and Python 3.13 or older; the rest use fakes, and a Python -child process in tshark's place. - -""" -from __future__ import annotations - -import importlib.util -import json -import os -import struct -import subprocess -import sys -import tempfile -import time -import types -import unittest -import warnings -from typing import TYPE_CHECKING -from unittest import mock - -from tests._support import close_extractor, reimport_once_per_class -from tests.foundation import _roundtrip as wire - -if TYPE_CHECKING: - from typing import Any, Callable - -RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') -HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) - - -def options(nxt: int) -> bytes: - """A Hop-by-Hop or Destination Options header of 8 octets, one PadN option in it.""" - return bytes([nxt, 0, 1, 4, 0, 0, 0, 0]) - - -def ah(nxt: int, length: int = 4) -> bytes: - """An Authentication Header whose Payload Length is ``length``, its ICV filled in.""" - fixed = bytes([nxt, length, 0, 0]) + (0x100).to_bytes(4, 'big') + (7).to_bytes(4, 'big') - return fixed + b'\xaa' * max((length + 2) * 4 - len(fixed), 0) - - -def fragment(nxt: int, offset: int = 0) -> bytes: - """An IPv6 Fragment header; ``offset`` is in octets.""" - return bytes([nxt, 0]) + offset.to_bytes(2, 'big') + (0xDEADBEEF).to_bytes(4, 'big') - - -def mobility(nxt: int, length: int = 0) -> bytes: - """A Mobility header, a Binding Refresh Request, whose Header Len is ``length``.""" - return bytes([nxt, length, 0, 0, 0, 0, 0, 0]) - - -def hip(nxt: int) -> bytes: - """A HIP header of 40 octets, an I1 with no parameters.""" - return bytes([nxt, 4, 1, 0x21, 0, 0, 0, 0]) + b'\x11' * 32 - - -def shim6(nxt: int) -> bytes: - """A Shim6 payload extension header of 8 octets.""" - return bytes([nxt, 0, 0x80, 0, 0, 0, 0, 1]) - - -def tcp(number: int, *, opts: bytes = b'', words: 'int | None' = None, **flags: bool) -> bytes: - """A TCP segment carrying ``data`` on a port pair of its own, with ``opts`` and a Data Offset of ``words``.""" - segment = wire.tcp(b'data', seq=1000 + number, ack=1, sport=1000 + number, **flags) - words = 5 + len(opts) // 4 if words is None else words - return segment[:12] + bytes([words << 4]) + segment[13:20] + opts + segment[20:] - - -def v4(payload: bytes, proto: int, *, length: 'int | None' = None, opts: bytes = b'', - **fields: 'Any') -> bytes: - """An IPv4 packet carrying ``payload`` as ``proto``, with options ``opts``, its Total Length ``length`` if given.""" - packet = bytearray(wire.ipv4(payload, proto=proto, **fields)) - if opts: - packet[0] = 0x40 | (5 + len(opts) // 4) - packet[2:4] = (len(packet) + len(opts)).to_bytes(2, 'big') - packet[20:20] = opts - if length is not None: - packet[2:4] = length.to_bytes(2, 'big') - packet[10:12] = b'\x00\x00' - packet[10:12] = wire.checksum(bytes(packet[:(packet[0] & 0x0F) * 4])).to_bytes(2, 'big') - return bytes(packet) - - -def v6(payload: bytes, nxt: int) -> bytes: - """An IPv6 packet carrying ``payload`` as ``nxt``.""" - return wire.ipv6(payload, nxt=nxt) - - -def eth(packet: bytes) -> bytes: - """An Ethernet frame carrying the IPv4 or IPv6 ``packet``.""" - return wire.ethernet(packet, 0x86DD if packet[0] >> 4 == 6 else 0x0800) - - -#: An MSS option of length 3, which the default engine's TCP parser rejects. -BAD_MSS = b'\x02\x03\x05\xb4' -#: An IPv4 Timestamp option of length 2, which its IPv4 parser rejects. -BAD_TIMESTAMP = b'\x44\x02\x00\x00' - -#: Frames carrying TCP behind an AH or IPv6 extension header in IPv4, each with -#: whether the default engine traces the TCP, by a name for it. Each builder -#: takes the TCP segment's number, which is its frame number. -CASES = { - 'control': (True, lambda n: eth(v4(tcp(n), 6))), - # #1625's six, the first two as controls - 'hop-by-hop': (True, lambda n: eth(v4(options(6) + tcp(n), 0))), - 'hop-by-hop, rejected tcp options': (False, lambda n: eth(v4(options(6) + tcp(n, opts=BAD_MSS), 0))), - 'hop-by-hop, tcp data offset past the capture': (False, lambda n: eth(v4(options(6) + tcp(n, words=15), 0))), - 'mobility': (True, lambda n: eth(v4(mobility(6) + tcp(n), 135))), - 'hip': (True, lambda n: eth(v4(hip(6) + tcp(n), 139))), - # the other headers - 'destination options': (True, lambda n: eth(v4(options(6) + tcp(n), 60))), - 'routing': (True, lambda n: eth(v4(bytes([6, 0, 0, 0, 0, 0, 0, 0]) + tcp(n), 43))), - 'fragment': (True, lambda n: eth(v4(fragment(6) + tcp(n), 44))), - # the default engine reads past an IPv6 Fragment header inside IPv4 whatever its offset - 'later fragment header': (True, lambda n: eth(v4(fragment(6, 40) + tcp(n), 44))), - 'ah': (True, lambda n: eth(v4(ah(6) + tcp(n), 51))), - 'ah without icv': (True, lambda n: eth(v4(ah(6, 1) + tcp(n), 51))), - # a Payload Length of 0 declares 8 octets, fewer than the fixed header - 'ah too short': (False, lambda n: eth(v4(ah(6, 0) + tcp(n), 51))), - 'mobility cut short': (False, lambda n: eth(v4(mobility(6, 1) + tcp(n)[:4], 135))), - # Shim6, whose parser refuses IPv4, so that the default engine reads it raw - 'shim6': (False, lambda n: eth(v4(shim6(6) + tcp(n), 140))), - 'shim6 in 4in4': (False, lambda n: eth(v4(v4(shim6(6) + tcp(n), 140), 4))), - 'shim6 in 4in6': (False, lambda n: eth(v6(v4(shim6(6) + tcp(n), 140), 4))), - 'hop-by-hop then shim6': (False, lambda n: eth(v4(options(140) + shim6(6) + tcp(n), 0))), - # but not IPv6, which reads past it - 'shim6 in 6in4 behind hop-by-hop': (True, lambda n: eth(v4(options(41) + v6(shim6(6) + tcp(n), 140), 0))), - # chains - 'ah then hop-by-hop': (True, lambda n: eth(v4(ah(0) + options(6) + tcp(n), 51))), - 'hop-by-hop then ah': (True, lambda n: eth(v4(options(51) + ah(6) + tcp(n), 0))), - 'hop-by-hop then destination options': (True, lambda n: eth(v4(options(60) + options(6) + tcp(n), 0))), - 'mobility then hop-by-hop': (True, lambda n: eth(v4(mobility(0) + options(6) + tcp(n), 135))), - 'hip then destination options': (True, lambda n: eth(v4(hip(60) + options(6) + tcp(n), 139))), - 'chain at the limit': (True, lambda n: eth(v4(b''.join(options(60 if i < 31 else 6) for i in range(32)) - + tcp(n), 60))), - 'chain past the limit': (False, lambda n: eth(v4(b''.join(options(60 if i < 32 else 6) for i in range(33)) - + tcp(n), 60))), - # tunnels behind the headers, and the headers inside tunnels - '4in4 behind hop-by-hop': (True, lambda n: eth(v4(options(4) + v4(tcp(n), 6), 0))), - '6in4 behind hop-by-hop': (True, lambda n: eth(v4(options(41) + v6(tcp(n), 6), 0))), - '6in4 behind ah': (True, lambda n: eth(v4(ah(41) + v6(tcp(n), 6), 51))), - '4in4 behind mobility': (True, lambda n: eth(v4(mobility(4) + v4(tcp(n), 6), 135))), - '6in4 behind hip': (True, lambda n: eth(v4(hip(41) + v6(tcp(n), 6), 139))), - 'hop-by-hop in 4in4': (True, lambda n: eth(v4(v4(options(6) + tcp(n), 0), 4))), - 'hop-by-hop in 4in6': (True, lambda n: eth(v6(v4(options(6) + tcp(n), 0), 4))), - 'mobility in 4in4': (True, lambda n: eth(v4(v4(mobility(6) + tcp(n), 135), 4))), - 'hop-by-hop in 4in4 behind hop-by-hop': (True, lambda n: eth(v4(options(4) + v4(options(6) + tcp(n), 0), 0))), - # IP behind the header that the default engine does not read - 'short ipv4 behind hop-by-hop': (False, lambda n: eth(v4(options(4) + v4(tcp(n), 6)[:12], 0))), - 'short ipv6 behind hop-by-hop': (False, lambda n: eth(v4(options(41) + v6(tcp(n), 6)[:30], 0))), - 'bad ipv4 behind hop-by-hop': (False, lambda n: eth(v4(options(4) + b'\x55' + v4(tcp(n), 6)[1:], 0))), - 'rejected ipv4 options behind hop-by-hop': (False, lambda n: eth(v4(options(4) + v4(tcp(n), 6, opts=BAD_TIMESTAMP), - 0))), - 'rejected ipv4 options, then hop-by-hop': (False, lambda n: eth(v4(options(6) + tcp(n), 0, opts=BAD_TIMESTAMP))), - # TCP behind the header, read as the TCP parser reads it - 'tcp options': (True, lambda n: eth(v4(options(6) + tcp(n, opts=b'\x02\x04\x05\xb4'), 0))), - 'tcp data offset below 5': (False, lambda n: eth(v4(options(6) + tcp(n, words=4), 0))), - 'tcp cut short': (False, lambda n: eth(v4(options(6) + tcp(n)[:16], 0))), - 'mobility, rejected tcp options': (False, lambda n: eth(v4(mobility(6) + tcp(n, opts=BAD_MSS), 135))), - 'hip, tcp data offset past the capture': (False, lambda n: eth(v4(hip(6) + tcp(n, words=15), 139))), - # directly over the inner IP layer, the default engine reads a rejected header raw - 'rejected tcp options in 4in4 behind hop-by-hop': (True, lambda n: eth(v4(options(4) + v4(tcp(n, opts=BAD_MSS), 6), - 0))), - 'rejected tcp options in 6in4 behind mobility': (True, lambda n: eth(v4(mobility(41) + v6(tcp(n, opts=BAD_MSS), 6), - 135))), - 'no tcp': (False, lambda n: eth(v4(options(17) + wire.udp(b'data', sport=1000 + n), 0))), - 'flags': (True, lambda n: eth(v4(mobility(6) + tcp(n, syn=True, fin=True, rst=True), 135))), - # the Total Length bounds what is read, and a zero one does not - 'total length short of the tcp': (False, lambda n: eth(v4(options(6) + tcp(n), 0, length=40))), - 'total length under the header': (False, lambda n: eth(v4(options(6) + tcp(n), 0, length=12))), - 'zero total length': (True, lambda n: eth(v4(options(6) + tcp(n), 0, length=0))), - 'trailer': (True, lambda n: eth(v4(mobility(6) + tcp(n), 135)) + b'\x00' * 10), - 'later ipv4 fragment': (False, lambda n: eth(v4(mobility(6) + tcp(n), 135, offset=8))), - 'first ipv4 fragment': (True, lambda n: eth(v4(mobility(6) + tcp(n), 135, mf=True))), - # the default engine does not dissect the datagram an ICMP error quotes - 'quoted in an icmp error': (False, lambda n: eth(v4(bytes([3, 3, 0, 0, 0, 0, 0, 0]) - + v4(mobility(6) + tcp(n), 135), 1))), -} - - -def gre(packet: bytes) -> bytes: - """A GRE packet, with no options, carrying the IPv4 ``packet``.""" - return b'\x00\x00\x08\x00' + packet - - -def vxlan(frame: bytes) -> bytes: - """A UDP datagram to the VXLAN port, carrying the Ethernet ``frame`` with VNI 1.""" - return wire.udp(b'\x08\x00\x00\x00\x00\x00\x01\x00' + frame, sport=4000, dport=4789) - - -def chain(count: int, nxt: int) -> bytes: - """``count`` Destination Options headers in a row, the last naming ``nxt``.""" - return b''.join(options(60 if index < count - 1 else nxt) for index in range(count)) - - -#: Frames whose AH or IPv6 extension header in IPv4 sits inside a tunnel or -#: behind an IPv6 header, as in :data:`CASES`. The default engine reads a frame -#: from its outermost IP layer on, so it stops at a tunnel it does not dissect, -#: such as GRE or VXLAN, which tshark does, and at a chain of more than -#: :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` extension -#: headers in the IPv6 header carrying the IPv4. -OUTERMOST_CASES = { - 'mobility in gre': (False, lambda n: eth(v4(gre(v4(mobility(6) + tcp(n), 135)), 47))), - 'hip in gre': (False, lambda n: eth(v4(gre(v4(hip(6) + tcp(n), 139)), 47))), - 'hop-by-hop in gre': (False, lambda n: eth(v4(gre(v4(options(6) + tcp(n), 0)), 47))), - 'mobility in 4in4 in gre': (False, lambda n: eth(v4(gre(v4(v4(mobility(6) + tcp(n), 135), 4)), 47))), - 'hop-by-hop in 4in4 in gre': (False, lambda n: eth(v4(gre(v4(v4(options(6) + tcp(n), 0), 4)), 47))), - 'mobility in vxlan': (False, lambda n: eth(v4(vxlan(eth(v4(mobility(6) + tcp(n), 135))), 17))), - 'hip in vxlan': (False, lambda n: eth(v4(vxlan(eth(v4(hip(6) + tcp(n), 139))), 17))), - 'hop-by-hop in vxlan': (False, lambda n: eth(v4(vxlan(eth(v4(options(6) + tcp(n), 0))), 17))), - 'mobility in 4in6 past the chain limit': (False, lambda n: eth(v6(chain(33, 4) + v4(mobility(6) + tcp(n), 135), - 60))), - 'hop-by-hop in 4in6 past the chain limit': (False, lambda n: eth(v6(chain(33, 4) + v4(options(6) + tcp(n), 0), - 60))), - 'mobility in 4in6 at the chain limit': (True, lambda n: eth(v6(chain(32, 4) + v4(mobility(6) + tcp(n), 135), 60))), - 'mobility in 4in6 behind a short chain': (True, lambda n: eth(v6(chain(2, 4) + v4(mobility(6) + tcp(n), 135), 60))), - 'hip in 4in6 in 6in4 behind hop-by-hop': (True, lambda n: eth(v4(options(41) + v6(v4(hip(6) + tcp(n), 139), 4), - 0))), -} - - -def vlan(packet: bytes, *tags: int) -> bytes: - """An Ethernet frame carrying the IPv4 ``packet`` behind a VLAN tag of each TPID in ``tags``.""" - inner = b''.join(struct.pack('>HH', 1, tpid) for tpid in tags[1:]) + struct.pack('>HH', 1, 0x0800) - return wire.ethernet(inner + packet, tags[0]) - - -def loopback(packet: bytes, family: int, byteorder: str) -> bytes: - """A BSD loopback frame carrying ``packet`` as the address ``family``, written in ``byteorder``.""" - return family.to_bytes(4, byteorder) + packet - - -def sll(packet: bytes) -> bytes: - """A Linux cooked-mode (SLL) frame carrying the IPv4 ``packet``.""" - return struct.pack('>HHH8sH', 0, 1, 6, b'\x02\x00\x00\x00\x00\x01', 0x0800) + packet - - -#: Frames of each link type that carry an AH or IPv6 extension header in IPv4, -#: each with the link type, whether the default engine traces the TCP, and a -#: builder as in :data:`CASES`, by a name for it. The default engine dissects IP -#: over the link types and EtherTypes :func:`pcapkit.toolkit.pyshark._network_start` -#: names only; a frame over any other, such as MPLS, Linux SLL, or the BSD raw -#: IP of link types 12 and 14, which tshark reads as raw IP (101), is not -#: traced (#1639). -LINK_CASES = { - 'vlan, mobility': (1, True, lambda n: vlan(v4(mobility(6) + tcp(n), 135), 0x8100)), - 'qinq, hip': (1, True, lambda n: vlan(v4(hip(6) + tcp(n), 139), 0x88A8, 0x8100)), - 'qinq, hop-by-hop, rejected tcp options': (1, False, lambda n: vlan(v4(options(6) + tcp(n, opts=BAD_MSS), 0), - 0x88A8, 0x8100)), - 'mpls, mobility': (1, False, lambda n: wire.ethernet(b'\x00\x01\x01\x40' + v4(mobility(6) + tcp(n), 135), 0x8847)), - 'null, mobility': (0, True, lambda n: loopback(v4(mobility(6) + tcp(n), 135), 2, 'little')), - 'null, mobility in 4in6': (0, True, lambda n: loopback(v6(v4(mobility(6) + tcp(n), 135), 4), 30, 'big')), - 'null, hop-by-hop, rejected tcp options': (0, False, lambda n: loopback(v4(options(6) + tcp(n, opts=BAD_MSS), 0), - 2, 'little')), - 'loop, hip': (108, True, lambda n: loopback(v4(hip(6) + tcp(n), 139), 2, 'big')), - 'loop, hop-by-hop in 4in6': (108, True, lambda n: loopback(v6(v4(options(6) + tcp(n), 0), 4), 24, 'big')), - 'raw, mobility': (101, True, lambda n: v4(mobility(6) + tcp(n), 135)), - 'raw, mobility in 4in6 past the chain limit': (101, False, lambda n: v6(chain(33, 4) - + v4(mobility(6) + tcp(n), 135), 60)), - 'ipv4, hip': (228, True, lambda n: v4(hip(6) + tcp(n), 139)), - 'ipv4, hop-by-hop, tcp data offset past the capture': (228, False, lambda n: v4(options(6) + tcp(n, words=15), 0)), - 'ipv6, mobility in 4in6': (229, True, lambda n: v6(v4(mobility(6) + tcp(n), 135), 4)), - 'sll, mobility': (113, False, lambda n: sll(v4(mobility(6) + tcp(n), 135))), - 'sll, hip': (113, False, lambda n: sll(v4(hip(6) + tcp(n), 139))), - 'bsd raw (12), mobility': (12, False, lambda n: v4(mobility(6) + tcp(n), 135)), - 'bsd raw (12), hip in 4in6': (12, False, lambda n: v6(v4(hip(6) + tcp(n), 139), 4)), - 'openbsd raw (14), hip': (14, False, lambda n: v4(hip(6) + tcp(n), 139)), - 'openbsd raw (14), mobility in 4in6': (14, False, lambda n: v6(v4(mobility(6) + tcp(n), 135), 4)), -} - - -def build(cases: 'dict[str, tuple[bool, Callable[[int], bytes]]]') -> 'tuple[list[bytes], list[int]]': - """The frames of ``cases``, and the numbers of those the default engine traces TCP from.""" - numbered = list(enumerate(cases.values(), start=1)) - return ([make(number) for number, (_, make) in numbered], - [number for number, (reads, _) in numbered if reads]) - - -def block(kind: int, body: bytes, byteorder: str = 'little') -> bytes: - """A PCAP-NG block of type ``kind`` around ``body``, zero-padded to 32 bits, in ``byteorder``.""" - order = '<' if byteorder == 'little' else '>' - body += bytes(-len(body) % 4) - return struct.pack(f'{order}II', kind, len(body) + 12) + body + struct.pack(f'{order}I', len(body) + 12) - - -def journal(entry: bytes) -> bytes: - """A systemd Journal Export Block holding ``entry``.""" - return block(9, entry) - - -def spb(octets: bytes) -> bytes: - """A Simple Packet Block of ``octets``, which carries no timestamp.""" - return block(3, struct.pack(' bytes: - """A PCAP-NG section in ``byteorder``, with an interface of each of ``linktypes``, then ``packets``. - - Each packet is ``(kind, interface, octets)``: an Enhanced Packet Block (6), - an obsolete Packet Block (2), or a Simple Packet Block (3), which names no - interface. Any other kind is a block of that type around ``octets``. - - """ - order = '<' if byteorder == 'little' else '>' - out = [block(0x0A0D0D0A, struct.pack(f'{order}IHHq', 0x1A2B3C4D, 1, 0, -1), byteorder)] - out += [block(1, struct.pack(f'{order}HHI', linktype, 0, 0xFFFF), byteorder) for linktype in linktypes] - for kind, interface, octets in packets: - head = {6: struct.pack(f'{order}IIIII', interface, 0, 1, len(octets), len(octets)), - 2: struct.pack(f'{order}HHIIII', interface, 0, 0, 1, len(octets), len(octets)), - 3: struct.pack(f'{order}I', len(octets))}.get(kind, b'') - out.append(block(kind, head + octets, byteorder)) - return b''.join(out) - - -def tshark_children() -> 'dict[int, str] | None': - """The state of each ``tshark`` process this one started, by pid, as ``ps`` reports it; :data:`None` without ``ps``.""" - try: - listing = subprocess.run(['ps', '-e', '-o', 'pid=', '-o', 'ppid=', '-o', 'stat=', '-o', 'comm='], - capture_output=True, text=True, timeout=30, check=True).stdout - except (OSError, subprocess.SubprocessError): - return None - children = {} - for line in listing.splitlines(): - fields = line.split(None, 3) - if len(fields) == 4 and fields[1] == str(os.getpid()) and 'tshark' in fields[3]: - children[int(fields[0])] = fields[2] - return children - - -def read(test: unittest.TestCase, engine: str, capture: bytes, suffix: str = 'pcap') -> 'dict[str, Any]': - """Read ``capture`` with ``engine``, with TCP reassembly and flow tracing on. - - Returns: - The input flow tracing receives for each frame number -- every field but - ``frame``, ``header`` and ``payload``, which PyShark cannot supply -- - the traced flows' labels and frames, and how many inputs TCP - reassembly received, or :data:`None` if the engine declines it. - - Skips ``test`` unless ``engine`` ran. - - """ - from pcapkit import extract - from pcapkit.foundation.reassembly.reassembly import ReassemblyBase - from pcapkit.foundation.traceflow.traceflow import TraceFlowBase - from pcapkit.utilities.exceptions import UnsupportedCall - - if engine != 'default' and importlib.util.find_spec(engine) is None: - test.skipTest(f'{engine} not installed') - reassemble, trace = ReassemblyBase.__call__, TraceFlowBase.__call__ - seen = {'tcp': 0, 'trace': {}} # type: dict[str, Any] - - def spy_tcp(this: 'Any', packet: 'Any') -> 'Any': - seen['tcp'] += 1 - return reassemble(this, packet) - - def spy_trace(this: 'Any', packet: 'Any') -> 'Any': - seen['trace'][packet.index] = {key: value for key, value in packet.items() - if key not in ('frame', 'header', 'payload')} - return trace(this, packet) - - with tempfile.TemporaryDirectory() as tmp, warnings.catch_warnings(), \ - mock.patch.object(ReassemblyBase, '__call__', spy_tcp), \ - mock.patch.object(TraceFlowBase, '__call__', spy_trace): - warnings.simplefilter('ignore') - path = os.path.join(tmp, f'capture.{suffix}') - with open(path, 'wb') as file: - file.write(capture) - extractor = extract(fin=path, nofile=True, engine=engine, reassembly=True, tcp=True, - trace=True, trace_fout=os.path.join(tmp, 'trace'), trace_format='json') - try: - if extractor._exnam != engine: # pylint: disable=protected-access - test.skipTest(f'{engine} did not run') - seen['flows'] = [(flow.label, tuple(flow.index)) for flow in extractor.trace.tcp] - try: - declined = extractor.reassembly.tcp is None - except UnsupportedCall: - declined = True - if declined: - seen['tcp'] = None - finally: - close_extractor(extractor) - return seen - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class PySharkExtensionHeaderTraceTests(unittest.TestCase): - """The pyshark engine traces the TCP the default engine reads behind an extension header in IPv4, and no other.""" - - def setUp(self) -> None: - reimport_once_per_class(self) - - def agree(self, capture: bytes, reads: 'list[int]', names: 'dict[int, str]', - suffix: str = 'pcap') -> None: - base = read(self, 'default', capture, suffix) - # what the default engine reads, as the cases say - self.assertEqual(sorted(base['trace']), reads) - seen = read(self, 'pyshark', capture, suffix) - self.assertEqual([names[number] for number in sorted(set(seen['trace']) ^ set(base['trace']))], [], - 'frames traced by one of pyshark and the default engine alone') - for number, record in base['trace'].items(): - self.assertEqual(seen['trace'][number], record, names[number]) - self.assertEqual(seen['flows'], base['flows']) - # pyshark still declines TCP reassembly - self.assertIsNone(seen['tcp']) - self.assertEqual(base['tcp'], len(reads)) - - def test_tcp_behind_an_extension_header_in_ipv4_is_traced_as_the_default_engine_reads_it(self) -> None: - frames, reads = build(CASES) - self.agree(wire.pcap([(1, number, octets) for number, octets in enumerate(frames)]), - reads, dict(enumerate(CASES, start=1))) - - def test_the_frame_is_read_from_its_outermost_ip_layer(self) -> None: - # GRE, VXLAN and an outer IPv6 header's chain bound stop it, as they - # stop the default engine, wherever the extension header in IPv4 is - frames, reads = build(OUTERMOST_CASES) - self.agree(wire.pcap([(1, number, octets) for number, octets in enumerate(frames)]), - reads, dict(enumerate(OUTERMOST_CASES, start=1))) - - def test_each_link_type_is_read_as_the_default_engine_reads_it(self) -> None: - for linktype in sorted({link for link, _, _ in LINK_CASES.values()}): - cases = {name: (reads, make) for name, (link, reads, make) in LINK_CASES.items() if link == linktype} - with self.subTest(linktype=linktype): - frames, reads = build(cases) - self.agree(wire.pcap([(1, number, octets) for number, octets in enumerate(frames)], - linktype=linktype), reads, dict(enumerate(cases, start=1))) - - def test_the_octets_are_those_of_the_frame_past_a_record_that_is_no_packet(self) -> None: - # tshark numbers a journal block as a frame, the default engine counts - # packets only, and the second tshark pass is keyed by tshark's number - cases = {name: CASES[name] for name in ('control', 'mobility', 'hip, tcp data offset past the capture', - 'hop-by-hop', 'hop-by-hop, rejected tcp options', 'flags')} - frames, reads = build(cases) - head = wire.pcapng([]) - packets = [wire.pcapng([(1, number, octets)])[len(head):] for number, octets in enumerate(frames)] - entry = journal(b'__REALTIME_TIMESTAMP=1000000\nMESSAGE=hello\n') - capture = head + entry + packets[0] + entry + b''.join(packets[1:3]) + entry + b''.join(packets[3:]) - self.agree(capture, reads, dict(enumerate(cases, start=1)), suffix='pcapng') - - def test_a_simple_packet_block_is_traced(self) -> None: - # a Simple Packet Block carries no timestamp, which PyShark's JSON - # parser could not do without - cases = {name: CASES[name] for name in ('control', 'hop-by-hop', 'mobility', - 'hop-by-hop, rejected tcp options')} - frames, reads = build(cases) - head = wire.pcapng([]) - capture = head + wire.pcapng([(1, 0, frames[0])])[len(head):] + b''.join(spb(octets) for octets in frames[1:]) - self.agree(capture, reads, dict(enumerate(cases, start=1)), suffix='pcapng') - - def test_each_interface_is_read_by_its_own_link_type(self) -> None: - # a PCAP-NG frame's link type is its interface's, among those of its - # own section; tshark reads 12 and 14 as it reads raw IP (101) - def raw(n: int) -> bytes: - return v4(mobility(6) + tcp(n), 135) - - def raw6(n: int) -> bytes: - return v6(v4(hip(6) + tcp(n), 139), 4) - - # by section: its interfaces' link types, then its blocks, each a block - # type, an interface, the octets, and whether the default engine - # traces them, or None for a block that is no packet - sections = [ - ([1, 12, 101], 'little', [(6, 0, eth(raw(1)), True), - (6, 1, raw(2), False), - (9, 0, b'__REALTIME_TIMESTAMP=1000000\nMESSAGE=hello\n', None), - (6, 2, raw(3), True), - (3, 0, eth(raw6(4)), True)]), - ([14, 101], 'big', [(3, 0, raw6(5), False), - (6, 1, raw6(6), True), - (2, 0, raw(7), False), - (2, 1, v4(options(6) + tcp(8, opts=BAD_MSS), 0), False)]), - ] - capture = b''.join(section(linktypes, [packet[:3] for packet in packets], byteorder) - for linktypes, byteorder, packets in sections) - packets = [(f'section {index}, block {kind}, interface {interface}', reads) - for index, (_, _, blocks) in enumerate(sections, start=1) - for kind, interface, _, reads in blocks if reads is not None] - self.agree(capture, [number for number, (_, reads) in enumerate(packets, start=1) if reads], - {number: name for number, (name, _) in enumerate(packets, start=1)}, suffix='pcapng') - - def test_a_second_pass_tshark_rejects_falls_back_with_a_warning(self) -> None: - # the frames are traced as tshark dissects them, as before #1625 - from pcapkit.utilities.warnings import PySharkWarning - - cases = {name: CASES[name] for name in ('control', 'hop-by-hop', 'mobility', - 'hop-by-hop, rejected tcp options')} - frames, _ = build(cases) - capture = wire.pcap([(1, number, octets) for number, octets in enumerate(frames)]) - with mock.patch('pcapkit.toolkit.pyshark.OCTETS_FILTER', 'ip.proto =='), \ - mock.patch('pcapkit.foundation.engines.pyshark.warn') as warn: - seen = read(self, 'pyshark', capture) - self.assertEqual(sorted(seen['trace']), [1, 2, 4]) - messages = [call.args[0] for call in warn.call_args_list if call.args[1] is PySharkWarning] - self.assertEqual(len(messages), 1, messages) - self.assertIn('Frame 1: the second tshark pass', messages[0]) # the control frame's TCP asks first (#1639) - self.assertIn('exited with status', messages[0]) - - def test_an_early_close_leaves_no_tshark_behind(self) -> None: - from pcapkit import extract - - if tshark_children() is None: - self.skipTest('ps is not available') - frames = [(CASES['mobility'] if number % 5 == 0 else CASES['control'])[1](number) for number in range(1, 400)] - for how in ('__exit__', '_cleanup'): - with self.subTest(close=how), tempfile.TemporaryDirectory() as tmp, warnings.catch_warnings(): - warnings.simplefilter('ignore') - path = os.path.join(tmp, 'capture.pcap') - with open(path, 'wb') as file: - file.write(wire.pcap([(1, number, octets) for number, octets in enumerate(frames)])) - before = tshark_children() or {} - extractor = extract(fin=path, nofile=True, engine='pyshark', auto=False, tcp=True, trace=True, - trace_fout=os.path.join(tmp, 'trace'), trace_format='json') - try: - if extractor._exnam != 'pyshark': # pylint: disable=protected-access - self.skipTest('pyshark did not run') - for _ in range(30): - next(extractor) - self.assertIsNotNone(extractor._exeng._exraw, 'the second pass started') - self.assertEqual(len(set(tshark_children() or {}) - set(before)), 2) - finally: - if how == '__exit__': - extractor.__exit__(None, None, None) - else: - extractor._cleanup() # pylint: disable=protected-access - deadline = time.monotonic() + 3 - while True: - left = {pid: state for pid, state in (tshark_children() or {}).items() if pid not in before} - if not left or time.monotonic() > deadline: - break - time.sleep(0.1) - self.assertEqual(left, {}, 'tshark processes left behind, by pid, with their state') - - -class FakeField(str): - """A PyShark field: text, with the position and raw value tshark reports for it.""" - - def __new__(cls, show: str, pos: int = 0, raw_value: str = '') -> 'FakeField': - field = super().__new__(cls, show) - field.pos, field.raw_value = str(pos), raw_value - return field - - -class FakeLayer: - def __init__(self, layer_name: str, **fields: object) -> None: - self.layer_name = layer_name - self.field_names = tuple(fields) - for name, value in fields.items(): - setattr(self, name, value) - - -#: An Ethernet frame of IPv4 protocol 135, a Mobility header and TCP, as on the wire. -MOBILITY = eth(v4(mobility(6) + tcp(7, syn=True), 135)) - - -#: ``MOBILITY``'s IPv4 packet, as a raw IP frame. -RAW_MOBILITY = MOBILITY[14:] - - -def fake_packet(octets: bytes = MOBILITY, *, number: str = '3', start: int = 14) -> 'Any': - """A PyShark packet of a frame of the IPv4 ``octets`` from ``start`` on, as tshark dissects ``MOBILITY``. - - The frame is Ethernet, or raw IP if ``start`` is 0, as tshark names it. - - """ - ip = FakeLayer('ip', version=FakeField('4', start, '4'), proto=FakeField(str(octets[start + 9])), - src=FakeField('192.0.2.1', start + 12, octets[start + 12:start + 16].hex()), - dst=FakeField('198.51.100.2', start + 16, octets[start + 16:start + 20].hex())) - frame = FakeLayer('frame', encap_type='1' if start else '7', time_epoch='1.5') - return types.SimpleNamespace(number=number, frame_info=frame, sniff_timestamp='1.5', - layers=[frame, FakeLayer('eth' if start else 'raw'), ip, FakeLayer('mipv6')]) - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class ToolkitOctetsTests(unittest.TestCase): - """:func:`tcp_traceflow` reads such a frame from its octets, with no tshark.""" - - def setUp(self) -> None: - reimport_once_per_class(self) - - def test_the_segment_is_read_from_the_octets(self) -> None: - import ipaddress - - from pcapkit.toolkit import pyshark as toolkit - - asked = [] # type: list[Any] - packet = fake_packet() - data = toolkit.tcp_traceflow(packet, count=2, - captured=lambda p: asked.append(p) or toolkit.Captured(1, MOBILITY)) - self.assertEqual(asked, [packet]) - self.assertEqual((data.index, data.src, data.dst, data.srcport, data.dstport, data.seq, data.ack), - (2, ipaddress.ip_address('192.0.2.1'), ipaddress.ip_address('198.51.100.2'), - 1007, 9, 1007, 1)) - self.assertEqual((data.syn, data.fin, data.rst), (True, False, False)) - - def test_the_octets_are_read_as_the_default_engine_reads_them(self) -> None: - # every case, read from its outermost IP header on - from pcapkit.toolkit import pyshark as toolkit - - fields = ('src', 'dst', 'srcport', 'dstport', 'seq', 'ack', 'syn', 'fin', 'rst') - for cases in (CASES, OUTERMOST_CASES): - frames, _ = build(cases) - base = read(self, 'default', wire.pcap([(1, number, octets) for number, octets in enumerate(frames)])) - for number, (name, octets) in enumerate(zip(cases, frames), start=1): - version = {b'\x08\x00': 4, b'\x86\xdd': 6}[octets[12:14]] - with self.subTest(case=name), warnings.catch_warnings(): - warnings.simplefilter('ignore') # what the parsers warn of rejecting a header - # the IP header is the frame's second layer, after Ethernet - segment = toolkit._from_octets(version, octets[14:], 2) # pylint: disable=protected-access - record = base['trace'].get(number) - if record is None: - self.assertIsNone(segment) - else: - self.assertIsNotNone(segment) - self.assertEqual({field: getattr(segment, field) for field in fields}, - {field: record[field] for field in fields}) - - def test_the_outermost_ip_layer_is_found_as_the_default_engine_finds_it(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - # its version, where it starts, and its place in the frame (#1609) - packet4, packet6 = v4(tcp(1), 6), v6(tcp(1), 6) - for name, linktype, octets, found in ( - ('ethernet', 1, eth(packet4), (4, 14, 2)), - ('ethernet, ipv6', 1, eth(packet6), (6, 14, 2)), - ('vlan', 1, vlan(packet4, 0x8100), (4, 18, 3)), - ('qinq', 1, vlan(packet4, 0x88A8, 0x8100), (4, 22, 4)), - ('ethernet, arp', 1, wire.ethernet(packet4, 0x0806), None), - ('ethernet, mpls', 1, wire.ethernet(b'\x00\x01\x01\x40' + packet4, 0x8847), None), - ('ethernet cut short', 1, eth(packet4)[:13], None), - ('vlan tag cut short', 1, vlan(packet4, 0x8100)[:17], None), - ('null', 0, loopback(packet4, 2, 'little'), (4, 4, 2)), - ('null, big-endian', 0, loopback(packet6, 28, 'big'), (6, 4, 2)), - ('null, another family', 0, loopback(packet4, 7, 'little'), None), - ('null cut short', 0, b'\x02\x00\x00', None), - ('loop', 108, loopback(packet6, 24, 'big'), (6, 4, 2)), - # a LOOP family is in network byte order, never inferred - ('loop, little-endian', 108, loopback(packet4, 2, 'little'), None), - ('raw', 101, packet4, (4, 0, 1)), - ('raw, ipv6', 101, packet6, (6, 0, 1)), - ('raw, another version', 101, b'\x55' + packet4[1:], None), - ('raw, empty', 101, b'', None), - ('ipv4', 228, packet4, (4, 0, 1)), - ('ipv6', 229, packet6, (6, 0, 1)), - ('linux sll', 113, sll(packet4), None), - ): - with self.subTest(case=name): - self.assertEqual(toolkit._network_start(linktype, octets), found) # pylint: disable=protected-access - - def test_the_octets_are_read_from_the_outermost_ip_layer_tshark_dissects(self) -> None: - # an IPv6 layer above the IPv4 one, as tshark dissects 4in6 - import ipaddress - - from pcapkit.toolkit import pyshark as toolkit - - octets = eth(v6(v4(mobility(6) + tcp(7, syn=True), 135), 4)) - ipv6 = FakeLayer('ipv6', version=FakeField('6', 14, '6'), - src=FakeField('2001:db8::1', 22, octets[22:38].hex()), - dst=FakeField('2001:db8::2', 38, octets[38:54].hex())) - ip = FakeLayer('ip', version=FakeField('4', 54, '4'), proto=FakeField('135'), - src=FakeField('192.0.2.1', 66, octets[66:70].hex()), - dst=FakeField('198.51.100.2', 70, octets[70:74].hex())) - packet = fake_packet() - packet.layers[2:3] = [ipv6, ip] - data = toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(1, octets)) - self.assertEqual((data.src, data.dst, data.srcport, data.syn), - (ipaddress.ip_address('192.0.2.1'), ipaddress.ip_address('198.51.100.2'), 1007, True)) - - # nothing is read unless the outermost IP layer tshark dissects is - # the one the default engine does, in the same octets - moved = bytearray(octets) - moved[38] ^= 0xFF - for name, layers, frame in ( - ('another destination address', [ipv6, ip], bytes(moved)), - ('the ipv4 layer first', [ip, ipv6], octets), - ('the ipv4 layer alone', [ip], octets), - ('behind a vlan tag', [ipv6, ip], vlan(octets[14:], 0x8100)), - ): - with self.subTest(case=name): - packet.layers[2:-1] = layers - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p, f=frame: toolkit.Captured(1, f))) - - def test_the_link_type_is_the_capture_files(self) -> None: - # not the one tshark's frame.encap_type names: it reads 12 and 14 as - # raw IP (encap 7, 101), and the default engine leaves them raw - import ipaddress - - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit import pyshark as toolkit - - packet = fake_packet(RAW_MOBILITY, start=0) - data = toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(101, RAW_MOBILITY)) - self.assertEqual((data.src, data.srcport, data.syn), (ipaddress.ip_address('192.0.2.1'), 1007, True)) - # the record's link type is still tshark's, the same for each link type read - self.assertEqual(data.protocol, LinkType.RAW) - for linktype in (12, 14): - with self.subTest(linktype=linktype): - self.assertIsNone(toolkit.tcp_traceflow( - packet, captured=lambda p, l=linktype: toolkit.Captured(l, RAW_MOBILITY))) - # nor is an Ethernet frame read as raw IP, or the other way round - self.assertIsNone(toolkit.tcp_traceflow(fake_packet(), captured=lambda p: toolkit.Captured(101, MOBILITY))) - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(1, RAW_MOBILITY))) - - def test_a_link_type_the_default_engine_reads_no_ip_in_is_not_traced(self) -> None: - # the default engine reads no TCP there (#1639); an unknown link type - # of tshark's raises only once a segment is found - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit import pyshark as toolkit - from pcapkit.utilities.exceptions import MissingKeyError - - for linktype in (113, 276, 12, 14, 999): # Linux SLL and SLL2, BSD raw IP, then none at all - with self.subTest(linktype=linktype): - self.assertIsNone(toolkit.tcp_traceflow( - fake_packet(), captured=lambda p, l=linktype: toolkit.Captured(l, MOBILITY))) - # with no frame.encap_type, the root layer names the record's link type - packet = fake_packet() - del packet.frame_info.encap_type - packet.frame_info.field_names = ('time_epoch',) - captured = lambda p: toolkit.Captured(1, MOBILITY) # noqa: E731 - self.assertEqual(toolkit.tcp_traceflow(packet, captured=captured).protocol, LinkType.ETHERNET) - packet.layers[1] = FakeLayer('sll') - with self.assertRaises(MissingKeyError): - toolkit.tcp_traceflow(packet, captured=captured) - packet = fake_packet() - packet.frame_info.encap_type = '999' - packet.layers.append(FakeLayer('tcp', urgent_pointer='0', srcport='1', dstport='2', seq_raw='3', ack_raw='4', - flags_syn='True', flags_fin='False', flags_reset='False')) - with self.assertRaises(MissingKeyError): # the frame cannot be had, so tshark's segment is traced - toolkit.tcp_traceflow(packet, captured=lambda p: None) - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(113, MOBILITY))) - - def test_no_octets_leaves_tsharks_dissection(self) -> None: - # tshark dissects nothing past the Mobility header, so there is no TCP - from pcapkit.toolkit import pyshark as toolkit - - self.assertIsNone(toolkit.tcp_traceflow(fake_packet())) - self.assertIsNone(toolkit.tcp_traceflow(fake_packet(), captured=lambda p: None)) - - def test_octets_other_than_the_layers_are_not_read(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - packet = fake_packet() - moved = bytearray(MOBILITY) - moved[26] ^= 0xFF # not the source address tshark reports - for octets in (bytes(moved), MOBILITY[:30], b'\x00' * len(MOBILITY)): - with self.subTest(octets=octets[:34].hex()): - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p, o=octets: toolkit.Captured(1, o))) - packet.layers[2].version = FakeField('4', -1, '4') - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(1, MOBILITY))) - del packet.layers[2].version - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(1, MOBILITY))) - - def test_without_tcp_only_an_ipv4_extension_header_asks_for_the_octets(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - def refuse(packet: 'Any') -> 'Any': - raise AssertionError('octets asked for') - - for proto in ('6', '17', '41', '50', 'six'): - with self.subTest(proto=proto): - packet = fake_packet() - packet.layers[2].proto = FakeField(proto) - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=refuse)) - packet = fake_packet() - del packet.layers[2].proto - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=refuse)) - # an IP layer below the first TCP layer is not looked at - packet = fake_packet() - packet.layers.insert(2, FakeLayer('tcp')) - self.assertIsNone(toolkit.tcp_traceflow(packet, captured=refuse)) - # each of the headers, and Shim6, asks for them - for proto in ('0', '43', '44', '51', '60', '135', '139', '140'): - with self.subTest(proto=proto): - asked = [] # type: list[Any] - packet = fake_packet() - packet.layers[2].proto = FakeField(proto) - toolkit.tcp_traceflow(packet, captured=lambda p: asked.append(p)) - self.assertEqual(asked, [packet]) - - def test_the_filter_names_every_extension_header(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - from pcapkit.toolkit.pypcapfile import IPV4_EXTENSION_HEADERS - - self.assertEqual(toolkit.OCTETS_FILTER, 'tcp || ip.proto == 0 || ip.proto == 43 || ip.proto == 44 || ' - 'ip.proto == 51 || ip.proto == 60 || ip.proto == 135 || ' - 'ip.proto == 139 || ip.proto == 140 || vlan') - self.assertEqual(len(IPV4_EXTENSION_HEADERS), 7) - self.assertEqual(toolkit.OCTETS_PROTOCOLS - IPV4_EXTENSION_HEADERS, {140}) - - -def ek(*numbers: int) -> str: - """tshark's ``-T ek -x -J frame`` output for frames ``numbers``, each frame's octets its number.""" - lines = [] - for number in numbers: - lines.append(json.dumps({'index': {'_index': 'packets-1970-01-01'}})) - lines.append(json.dumps({'timestamp': '0', 'layers': { - 'frame_raw': bytes([number]).hex(), 'frame': {'frame_frame_number': str(number)}}})) - return ''.join(f'{line}\n' for line in lines) - - -def child(output: str = '', *, error: str = '', status: int = 0, linger: bool = False) -> 'list[str]': - """A command standing in for tshark: it writes ``output`` and ``error``, then exits with ``status``, or lingers.""" - script = (f'import sys, time\nsys.stdout.write({output!r})\nsys.stdout.flush()\n' - f'sys.stderr.write({error!r})\nsys.stderr.flush()\n' - + ('time.sleep(60)\n' if linger else '') + f'sys.exit({status})\n') - return [sys.executable, '-c', script] - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class EngineCapturedTests(unittest.TestCase): - """:meth:`PyShark._captured` reads the file and second pass forward, and falls back if either fails; no tshark.""" - - def setUp(self) -> None: - reimport_once_per_class(self) - - def capture(self, data: bytes, suffix: str = 'pcap') -> str: - """Write ``data`` to a capture file removed after the test, and return its path.""" - tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with - self.addCleanup(tmp.cleanup) - path = os.path.join(tmp.name, f'capture.{suffix}') - with open(path, 'wb') as file: - file.write(data) - return path - - def engine(self, command: 'list[str]', capture: 'str | None' = None) -> 'Any': - from pcapkit.foundation.engines.pyshark import PyShark - - engine = PyShark.__new__(PyShark) # skip __init__, which imports pyshark - engine._extractor = types.SimpleNamespace(_ifnm=capture or self.capture(wire.pcap([])), _frnum=7) - engine._extmp = types.SimpleNamespace(close=mock.Mock()) - engine._exraw = engine._exraw_err = engine._exraw_held = None - engine._exlink = engine._exlink_held = None - engine._exraw_failed = False - engine._octets_command = mock.Mock(return_value=command) - self.addCleanup(engine._stop_captured) - return engine - - @staticmethod - def ask(engine: 'Any', number: int, count: 'int | None' = None) -> 'Any': - if count is not None: - engine._extractor._frnum = count - return engine._captured(types.SimpleNamespace(number=str(number))) - - def test_the_second_pass_starts_on_the_first_request_and_reads_forward(self) -> None: - from pcapkit.utilities.warnings import PySharkWarning - - engine = self.engine(child(ek(2, 5, 9))) - engine._octets_command.assert_not_called() - with mock.patch('pcapkit.foundation.engines.pyshark.warn') as warn: - self.assertEqual(self.ask(engine, 2), (1, b'\x02')) - process = engine._exraw - self.assertEqual(self.ask(engine, 2), (1, b'\x02')) # the frame read last is kept - self.assertEqual(self.ask(engine, 5), (1, b'\x05')) - self.assertEqual(self.ask(engine, 9), (1, b'\x09')) - warn.assert_not_called() - self.assertIsNone(self.ask(engine, 10)) # past its last frame - self.assertIsNone(self.ask(engine, 11)) - engine._octets_command.assert_called_once_with() - self.assertEqual(warn.call_count, 1) - message, category = warn.call_args.args - self.assertIs(category, PySharkWarning) - self.assertIn('Frame 7: the second tshark pass', message) - self.assertIn('ended before frame 10', message) - self.assertIsNotNone(process.returncode) # reaped - self.assertIsNone(engine._exraw) - - def test_a_failing_second_pass_falls_back_with_a_warning(self) -> None: - from pcapkit.utilities.warnings import PySharkWarning - - missing = json.dumps({'layers': {'frame': {'frame_frame_number': '4'}}}) + '\n' - for name, command, number, reason in ( - ('exits with an error', child(error='tshark: "ip.proto ==" is not a valid filter\n', status=4), - 4, 'exited with status 4 (tshark: "ip.proto ==" is not a valid filter)'), - ('exits with an error and nothing said', child(status=2), 4, 'exited with status 2'), - ('writes what is not json', child('not json\n'), 4, "gave output it cannot parse: b'not json\\n'"), - ('writes no octets', child(missing), 4, 'gave output it cannot parse'), - ('lacks the frame', child(ek(2, 5)), 4, 'has no frame 4'), - ('cannot be started', ['/nonexistent/tshark'], 4, 'failed: FileNotFoundError'), - ): - with self.subTest(case=name), mock.patch('pcapkit.foundation.engines.pyshark.warn') as warn: - engine = self.engine(command) - self.assertIsNone(self.ask(engine, number)) - self.assertEqual(warn.call_count, 1) - message, category = warn.call_args.args - self.assertIs(category, PySharkWarning) - self.assertIn(reason, message) - self.assertIn('later such frames are traced as tshark dissects them', message) - self.assertIsNone(engine._exraw) - self.assertIsNone(engine._exraw_err) - # nothing more is asked of it, and nothing more is said - self.assertIsNone(self.ask(engine, number + 1)) - self.assertEqual(warn.call_count, 1) - self.assertEqual(engine._octets_command.call_count, 1) - - with mock.patch('pcapkit.foundation.engines.pyshark.warn') as warn: - engine = self.engine([]) - engine._octets_command.side_effect = LookupError('no tshark') - self.assertIsNone(self.ask(engine, 1)) - self.assertIn('failed: LookupError: no tshark', warn.call_args.args[0]) - - def test_close_reaps_the_second_pass(self) -> None: - engine = self.engine(child(ek(1), linger=True)) - self.assertEqual(self.ask(engine, 1), (1, b'\x01')) - process = engine._exraw - self.assertIsNone(process.poll()) - engine.close() - engine._extmp.close.assert_called_once_with() - self.assertIsNotNone(process.returncode) - self.assertTrue(process.stdout.closed) - self.assertIsNone(engine._exraw) - engine.close() # and again - - # the second pass is reaped even if the first will not close - engine = self.engine(child(ek(1), linger=True)) - self.ask(engine, 1) - process = engine._exraw - engine._extmp.close.side_effect = OSError('first pass') - with self.assertRaises(OSError): - engine.close() - self.assertIsNotNone(process.returncode) - - def test_the_link_type_is_read_from_the_capture_file_by_count(self) -> None: - # by the engine's count, as the default engine numbers frames, where - # the second pass is read by tshark's frame number - capture = self.capture(section([1, 12], [(6, 1, b'a'), (9, 0, b'x'), (3, 0, b'b'), (6, 1, b'c')]) - + section([101], [(2, 0, b'd')], 'big'), 'pcapng') - engine = self.engine(child(ek(2, 4, 5, 6)), capture) - with mock.patch('pcapkit.foundation.engines.pyshark.warn') as warn: - self.assertEqual(self.ask(engine, 2, count=1), (12, b'\x02')) - self.assertEqual(self.ask(engine, 2, count=1), (12, b'\x02')) # the packet read last is kept - self.assertEqual(self.ask(engine, 4, count=2), (1, b'\x04')) - self.assertEqual(self.ask(engine, 6, count=4), (101, b'\x06')) - warn.assert_not_called() - - def test_a_capture_file_that_cannot_be_read_falls_back_with_a_warning(self) -> None: - from pcapkit.utilities.warnings import PySharkWarning - - for name, data, count, reason in ( - ('lacks the packet', section([1], [(6, 0, b'a')]), 2, 'has no packet 2'), - ('names no interface', section([1], [(6, 1, b'a')]), 1, - 'failed: FormatError: PCAP-NG: [Block 6] invalid interface ID: 1'), - ('cannot be opened', None, 1, 'failed: FileNotFoundError'), - ): - with self.subTest(case=name), mock.patch('pcapkit.foundation.engines.pyshark.warn') as warn: - capture = os.path.join(tempfile.gettempdir(), 'no-such-dir-1625', 'x.pcapng') - engine = self.engine(child(ek(1, 2)), capture if data is None else self.capture(data, 'pcapng')) - self.assertIsNone(self.ask(engine, count, count=count)) - self.assertEqual(warn.call_count, 1) - message, category = warn.call_args.args - self.assertIs(category, PySharkWarning) - self.assertIn(f'Frame {count}: the capture file, read for the link type of a frame with an AH ' - 'or IPv6 extension header in IPv4,', message) - self.assertIn(reason, message) - self.assertIn('later such frames are traced as tshark dissects them', message) - self.assertIsNone(engine._exlink) - # nothing more is asked of either, and nothing more is said - self.assertIsNone(self.ask(engine, count + 1, count=count + 1)) - self.assertEqual(warn.call_count, 1) - engine._octets_command.assert_not_called() - - # once it has read the link type, the second pass's failure stops both - engine = self.engine(child(status=3), self.capture(section([1], [(6, 0, b'a')]), 'pcapng')) - with mock.patch('pcapkit.foundation.engines.pyshark.warn') as warn: - self.assertIsNone(self.ask(engine, 1, count=1)) - self.assertIn('the second tshark pass', warn.call_args.args[0]) - self.assertIsNone(engine._exlink) - - def test_close_closes_the_capture_file(self) -> None: - engine = self.engine(child(ek(1)), self.capture(section([1], [(6, 0, b'a')]), 'pcapng')) - self.assertEqual(self.ask(engine, 1, count=1), (1, b'\x01')) - file = engine._exlink.gi_frame.f_locals['file'] - self.assertFalse(file.closed) - engine.close() - self.assertTrue(file.closed) - self.assertIsNone(engine._exlink) - - def test_linktypes_are_read_as_the_default_engine_reads_them(self) -> None: - import itertools - - from pcapkit.foundation.engines.pyshark import _linktypes - from pcapkit.utilities.exceptions import FormatError - - def read(data: bytes, limit: int = 20) -> 'list[int]': - return list(itertools.islice(_linktypes(self.capture(data)), limit)) - - # a PCAP file's, of every frame, whatever its byte order or resolution - for options in ({}, {'byteorder': 'big'}, {'nanosecond': True}): - with self.subTest(pcap=options): - self.assertEqual(read(wire.pcap([], linktype=12, **options), 3), [12, 12, 12]) - - # a PCAP-NG packet's interface's, in its own section; other blocks are not packets - first = section([1, 12], [(6, 1, b'a'), (9, 0, b'x'), (3, 0, b'b'), (4, 0, b'n'), (2, 0, b'c'), - (5, 0, b's')]) - second = section([14, 101, 229], [(2, 2, b'd'), (6, 1, b'e'), (3, 0, b'f')], 'big') - self.assertEqual(read(first + second), [12, 1, 1, 229, 101, 14]) - self.assertEqual(read(first + second[:-3]), [12, 1, 1, 229, 101]) # a block cut short ends them - self.assertEqual(read(first + second[:10]), [12, 1, 1]) - self.assertEqual(read(section([1], [])), []) - - for name, data, message in ( - ('an interface the section does not describe', first + section([1], [(6, 1, b'a')]), - 'invalid interface ID: 1'), - ('the interface of a previous section', first + section([], [(3, 0, b'a')]), 'invalid interface ID: 0'), - ('an unknown byte order', section([1], [(6, 0, b'a')])[:8] + b'\x00\x00\x00\x00' - + section([1], [(6, 0, b'a')])[12:], 'block of no known byte order'), - ('a block shorter than 12 octets', section([1], []) + struct.pack(' None: - if importlib.util.find_spec('pyshark') is None: - self.skipTest('pyshark not installed') - from pcapkit.foundation.engines.pyshark import PyShark - from pcapkit.toolkit.pyshark import OCTETS_FILTER - - engine = PyShark.__new__(PyShark) # skip __init__, which imports pyshark - engine._extractor = types.SimpleNamespace(_ifnm='in.pcap') - with mock.patch('pyshark.tshark.tshark.get_process_path', return_value='/bin/tshark'): - self.assertEqual(engine._octets_command(), [ - '/bin/tshark', '-l', '-n', '-r', 'in.pcap', '-Y', OCTETS_FILTER, '-T', 'ek', '-x', '-J', 'frame', - '-o', 'ip.defragment:FALSE', '-o', 'ipv6.defragment:FALSE']) - - def test_ek_records_are_parsed(self) -> None: - from pcapkit.foundation.engines.pyshark import _ek_record - - index, packet = ek(2).splitlines() - self.assertIsNone(_ek_record(index.encode())) - self.assertEqual(_ek_record(packet.encode()), (2, b'\x02')) - listed = json.dumps({'layers': {'frame_raw': ['0a0b', 0, 2], 'frame': {'frame_frame_number': ['3']}}}) - self.assertEqual(_ek_record(listed.encode()), (3, b'\x0a\x0b')) - for line in (b'[]', b'{}', b'{"index": 1, "x": 2}', b'\xff', - b'{"layers": {"frame_raw": "zz", "frame": {"frame_frame_number": "1"}}}', - b'{"layers": {"frame_raw": 1, "frame": {"frame_frame_number": "1"}}}', - b'{"layers": {"frame_raw": "00", "frame": {"frame_frame_number": "one"}}}'): - with self.subTest(line=line): - self.assertIn('gave output it cannot parse', _ek_record(line)) - - def test_read_frame_hands_tcp_traceflow_the_captured_frame(self) -> None: - from pcapkit.toolkit import pyshark as toolkit - - engine = self.engine([]) - engine._extmp = types.SimpleNamespace(next=lambda: fake_packet(number='1')) - engine._extractor = types.SimpleNamespace(_frnum=0, _vfunc=lambda e, f: None, _flag_q=True, - _flag_d=False, _flag_t=True, _tcp=True, - _trace=types.SimpleNamespace(tcp=mock.Mock())) - with mock.patch.object(toolkit, 'tcp_traceflow', return_value=None) as traceflow: - engine.read_frame() - self.assertEqual(traceflow.call_args.kwargs, {'count': 1, 'captured': engine._captured}) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/toolkit/test_pyshark_unit.py b/tests/toolkit/test_pyshark_unit.py index 6a2a1fda5..dc28ee05c 100644 --- a/tests/toolkit/test_pyshark_unit.py +++ b/tests/toolkit/test_pyshark_unit.py @@ -680,7 +680,6 @@ def test_the_engine_passes_its_frame_count(self) -> None: engine = PyShark.__new__(PyShark) # skip __init__, which imports pyshark engine._extractor = extractor engine._extmp = types.SimpleNamespace(next=lambda: next(stream)) - engine._exraw_failed = True # no capture file to read frames from: traced as tshark dissects them (#1639) counts = [] while True: @@ -819,66 +818,81 @@ def test_a_whole_header_is_traced(self) -> None: @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class PySharkReassemblyPrefsTests(unittest.TestCase): - """#1612: the engine runs tshark with IPv4 and IPv6 reassembly off.""" +class PySharkAddressTests(unittest.TestCase): + """#1671: a TCP layer whose IP layer lacks an address is not traced. + + tshark reports no ``ip.dst`` behind an LSRR or SSRR option whose route data + is no whole number of addresses. Reading it raised a bare + :exc:`AttributeError` and aborted the whole extraction. + + """ def setUp(self) -> None: reimport_once_per_class(self) - def test_run_turns_tshark_ip_reassembly_off(self) -> None: - from pcapkit.foundation.engines.pyshark import PyShark + def test_an_ip_layer_without_an_address_is_skipped(self) -> None: + from pcapkit.toolkit import pyshark as toolkit - file_capture = mock.Mock(return_value='capture') - extractor = types.SimpleNamespace( - _exlyr='none', _exptl='null', _exctx=None, _flag_r=False, _ipv4=False, _ipv6=False, - _tcp=True, _flag_v=False, _ifnm='in.pcap', record_header=mock.Mock()) - engine = PyShark.__new__(PyShark) # skip __init__, which imports pyshark - engine._extractor = extractor - engine._expkg = types.SimpleNamespace(FileCapture=file_capture) - engine.run() - file_capture.assert_called_once_with( - 'in.pcap', keep_packets=False, - override_prefs={'ip.defragment': 'FALSE', 'ipv6.defragment': 'FALSE'}) - self.assertEqual(engine._extmp, 'capture') + for ipv6 in (False, True): + for missing in ('src', 'dst'): + with self.subTest(ipv6=ipv6, missing=missing): + packet = FakePySharkPacket(ipv6=ipv6, encap_type='1') + fields = {name: getattr(packet.layers[1], name) for name in ('src', 'dst') if name != missing} + packet.layers[1] = FakeLayer(packet.layers[1].layer_name, **fields) + self.assertIsNone(toolkit.tcp_traceflow(packet, count=3)) + + def test_an_ip_layer_with_both_addresses_is_traced(self) -> None: + import ipaddress + + from pcapkit.toolkit import pyshark as toolkit + + for ipv6, addresses in ((False, ('192.0.2.1', '198.51.100.1')), (True, ('2001:db8::1', '2001:db8::2'))): + with self.subTest(ipv6=ipv6): + data = toolkit.tcp_traceflow(FakePySharkPacket(ipv6=ipv6, encap_type='1'), count=3) + self.assertEqual((data.index, data.src, data.dst), + (3, *map(ipaddress.ip_address, addresses))) -def tcp_frame(version: int, segment: bytes, *, reverse: bool = False) -> bytes: +def tcp_frame(version: int, segment: bytes) -> bytes: """An Ethernet frame of an IPv4 or IPv6 packet carrying the TCP ``segment``.""" if version == 4: - return wire.ethernet(wire.ipv4(segment, proto=6, reverse=reverse), 0x0800, reverse=reverse) - return wire.ethernet(wire.ipv6(segment, nxt=6, reverse=reverse), 0x86DD, reverse=reverse) - - -def fragments(version: int, segment: bytes, ident: int, size: int = 24) -> 'list[bytes]': - """The TCP ``segment`` as one IPv4 or IPv6 datagram, cut into ``size``-octet fragments.""" - frames = [] - for offset in range(0, len(segment), size): - piece, more = segment[offset:offset + size], offset + size < len(segment) - if version == 4: - frames.append(wire.ethernet(wire.ipv4(piece, proto=6, ident=ident, offset=offset, mf=more), - 0x0800)) - else: - frames.append(wire.ethernet(wire.ipv6_fragment(piece, nxt=6, ident=ident, offset=offset, - mf=more), 0x86DD)) - return frames + return wire.ethernet(wire.ipv4(segment, proto=6), 0x0800) + return wire.ethernet(wire.ipv6(segment, nxt=6), 0x86DD) + + +def route_option(kind: int, length: int, data: bytes) -> bytes: + """An IPv4 source route option of ``kind`` claiming ``length`` octets, pointer 4, then ``data``, padded.""" + raw = bytes([kind, length, 4]) + data + return raw + bytes(-len(raw) % 4) -def read_trace(test: unittest.TestCase, engine: str, frames: 'list[bytes]') -> 'dict[str, Any]': - """Read ``frames`` with ``engine``, with TCP flow tracing on. +def with_options(packet: bytes, options: bytes) -> bytes: + """The IPv4 ``packet`` with ``options`` after its fixed header, its length fields and checksum updated.""" + header = bytearray(packet[:20] + options) + header[0] = 0x40 | len(header) // 4 + header[2:4] = (len(packet) + len(options)).to_bytes(2, 'big') + header[10:12] = b'\x00\x00' + header[10:12] = wire.checksum(bytes(header)).to_bytes(2, 'big') + return bytes(header) + packet[20:] + + +def read_trace(test: unittest.TestCase, frames: 'list[bytes]') -> 'dict[str, Any]': + """Read ``frames`` with the PyShark engine, with TCP flow tracing on. Returns: The input flow tracing receives for each frame number -- every field but ``frame``, ``header`` and ``payload``, which PyShark cannot supply -- and the traced flows' labels and frames. - Skips ``test`` unless ``engine`` ran, which for PyShark needs ``pyshark``, - the ``tshark`` binary and Python 3.13 or older. + Skips ``test`` unless the engine ran, which needs ``pyshark``, the + ``tshark`` binary and Python 3.13 or older. """ from pcapkit import extract from pcapkit.foundation.traceflow.traceflow import TraceFlowBase - if engine != 'default' and importlib.util.find_spec(engine) is None: + engine = 'pyshark' + if importlib.util.find_spec(engine) is None: test.skipTest(f'{engine} not installed') trace = TraceFlowBase.__call__ # type: Callable[..., Any] seen = {} # type: dict[int, dict[str, Any]] @@ -907,7 +921,7 @@ def spy(this: 'Any', packet: 'Any') -> 'Any': @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') class PySharkTsharkTraceTests(unittest.TestCase): - """The PyShark engine traces what the default engine does, read by a real tshark. + """What the PyShark engine traces, read by a real tshark. Every capture is built here. Each test skips unless the PyShark engine runs. @@ -916,14 +930,6 @@ class PySharkTsharkTraceTests(unittest.TestCase): def setUp(self) -> None: reimport_once_per_class(self) - def agree(self, frames: 'list[bytes]') -> 'dict[str, Any]': - """Check PyShark traces ``frames`` as the default engine does, frame by frame.""" - base = read_trace(self, 'default', frames) - seen = read_trace(self, 'pyshark', frames) - self.assertEqual(seen['trace'], base['trace']) - self.assertEqual(seen['flows'], base['flows']) - return base - def test_a_tcp_header_tshark_cannot_dissect_whole_is_skipped(self) -> None: # #1611: one good flow, then a segment with each Data Offset, then the # same segment cut at each octet of its header and at its end @@ -937,47 +943,28 @@ def test_a_tcp_header_tshark_cannot_dissect_whole_is_skipped(self) -> None: for words in range(16)] whole = tcp_frame(version, segment) cuts = [whole[:len(whole) - len(segment) + octets] for octets in range(21)] - base = self.agree(good + offsets + cuts) + seen = read_trace(self, good + offsets + cuts) # frames 4-19 are Data Offsets 0-15, and 20-40 cuts at 0-20 octets - self.assertEqual(sorted(base['trace']), [1, 2, 3, *range(9, 20), 40]) - - def test_fragmented_tcp_is_traced_at_its_first_fragment(self) -> None: - # #1612: tshark traced it at the fragment that completes the datagram - for version in (4, 6): - with self.subTest(ip=version): - frames = [] - for ident, sport in ((0x100, 40000), (0x101, 40001)): - frames.append(tcp_frame(version, wire.tcp(b'', seq=100, syn=True, sport=sport))) - frames += fragments(version, wire.tcp(wire.payload_pattern(60), seq=101, ack=1, - sport=sport), ident) - base = self.agree(frames) - self.assertEqual([index for _, index in base['flows']], [(1, 2), (6, 7)]) - - def test_a_stream_around_fragmented_segments_is_traced_as_by_the_default_engine(self) -> None: - # #1612: with its IP reassembly off, tshark sizes a fragmented segment - # from its first fragment, so its sequence analysis flags a lost - # segment on the later frames of the stream. tcp_traceflow reads none - # of those analysis fields. - for version in (4, 6): - with self.subTest(ip=version): - frames = [ - tcp_frame(version, wire.tcp(b'', seq=100, syn=True)), - tcp_frame(version, wire.tcp(b'', seq=500, ack=101, syn=True, sport=9, dport=40000), - reverse=True), - tcp_frame(version, wire.tcp(b'', seq=101, ack=501)), - *fragments(version, wire.tcp(wire.payload_pattern(100), seq=101, ack=501), 0x100, - size=32), - tcp_frame(version, wire.tcp(b'', seq=501, ack=201, sport=9, dport=40000), reverse=True), - tcp_frame(version, wire.tcp(b'abc', seq=201, ack=501)), - tcp_frame(version, wire.tcp(b'', seq=501, ack=204, sport=9, dport=40000), reverse=True), - *fragments(version, wire.tcp(wire.payload_pattern(100, seed=2), seq=204, ack=501), 0x101, - size=32), - tcp_frame(version, wire.tcp(b'xyz', seq=304, ack=501)), - tcp_frame(version, wire.tcp(b'', seq=307, ack=501, fin=True)), - ] - base = self.agree(frames) - # every frame but the later fragments, 5-7 and 12-14 - self.assertEqual(sorted(base['trace']), [1, 2, 3, 4, 8, 9, 10, 11, 15, 16]) + self.assertEqual(sorted(seen['trace']), [1, 2, 3, *range(9, 20), 40]) + self.assertEqual([index for _, index in seen['flows']], [(1, 2, 3), (*range(9, 20), 40)]) + + def test_a_route_option_tshark_reports_no_destination_behind_is_skipped(self) -> None: + # #1671: route data that is no whole number of addresses, in the IPv4 + # header carrying the segment, directly or inside 4in4; the extraction + # reads on past them + hop = bytes([10, 0, 0, 1]) + segment = wire.tcp(b'data', seq=7, ack=1, sport=50000) + malformed = [route_option(kind, length, data) for kind, length, data in ( + (0x83, 9, hop + b'\x01\x02'), (0x89, 9, hop + b'\x01\x02'), (0x83, 5, b'\x01\x02'), + (0x83, 13, hop + bytes(6)))] + frames = [tcp_frame(4, wire.tcp(b'', seq=100, syn=True)), + *(wire.ethernet(with_options(wire.ipv4(segment, proto=6), option), 0x0800) for option in malformed), + wire.ethernet(wire.ipv4(with_options(wire.ipv4(segment, proto=6), malformed[0]), proto=4), 0x0800), + tcp_frame(4, wire.tcp(b'hello', seq=101, ack=1)), + tcp_frame(4, wire.tcp(b'', seq=106, ack=1, fin=True))] + seen = read_trace(self, frames) + self.assertEqual(sorted(seen['trace']), [1, 7, 8]) + self.assertEqual([index for _, index in seen['flows']], [(1, 7, 8)]) if __name__ == '__main__':