From d74af19ad44d195236a41bb19872094abdb4d10b Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 14 Oct 2026 13:42:54 -0400 Subject: [PATCH] feat(utilities)!: one config module for the tunable limits (#1726) Every tunable limit and the vendor crawlers' retry and CI settings move to the new pcapkit.utilities.config, each read once from a PCAPKIT_* variable. Readers look them up as config. at call time, so one patch reaches every engine. A bad value warns with the new ConfigWarning, naming the variable and the value, and falls back; -1 lifts a limit, and that or a value above the default warns of the risk. The proxy variables are still read on every call, through config.http_proxy() and https_proxy(). A bad PCAPKIT_DEVMODE, PCAPKIT_VERBOSE or PCAPKIT_SPHINX is recorded by utilities.logging and reported by utilities.warnings once it has loaded. Both flag readers strip whitespace, and a ConfigWarning given at import names the importing module rather than :0. The docs extra now needs Sphinx>=7.3, for versionremoved. BREAKING CHANGE: the old homes are removed, with no re-export: protocols.protocol.FRAME_LAYER_LIMIT, protocols.internet.internet.EXTENSION_HEADER_LIMIT, toolkit.pypcapfile.IPV6_TCP_WARNED_LIMIT, dumpkit.common.FALLBACK_LIMIT, utilities.chardet.DETECT_CACHE_SIZE, corekit.io._READ_CHUNK, the corekit.fields.field zero-pad guards, and vendor.default.MAX_RETRY and CI_MODE. Refs #1726 --- .github/workflows/unit-tests.yml | 2 +- docs/source/contributing/pep.rst | 6 +- docs/source/pcapkit/corekit/fields/field.rst | 14 + docs/source/pcapkit/corekit/io.rst | 6 + docs/source/pcapkit/dumpkit/common.rst | 6 + docs/source/pcapkit/index.rst | 25 +- .../pcapkit/protocols/internet/internet.rst | 14 +- docs/source/pcapkit/protocols/protocol.rst | 10 +- docs/source/pcapkit/toolkit/3rdparty.rst | 6 + docs/source/pcapkit/utilities/chardet.rst | 8 +- docs/source/pcapkit/utilities/config.rst | 160 +++++ docs/source/pcapkit/utilities/index.rst | 1 + docs/source/pcapkit/utilities/warnings.rst | 4 + docs/source/pcapkit/vendor/default.rst | 11 + pcapkit/corekit/fields/field.py | 150 ++-- pcapkit/corekit/io.py | 18 +- pcapkit/dumpkit/common.py | 20 +- pcapkit/protocols/internet/internet.py | 52 +- pcapkit/protocols/internet/ipv6.py | 14 +- pcapkit/protocols/protocol.py | 46 +- pcapkit/toolkit/pcap.py | 4 +- pcapkit/toolkit/pypcapfile.py | 29 +- pcapkit/toolkit/scapy.py | 5 +- pcapkit/utilities/chardet.py | 17 +- pcapkit/utilities/config.py | 419 +++++++++++ pcapkit/utilities/logging.py | 36 +- pcapkit/utilities/warnings.py | 73 +- pcapkit/vendor/default.py | 42 +- pcapkit/vendor/pcapng/block_type.py | 2 +- pyproject.toml | 2 +- tests/corekit/test_fields_field.py | 59 +- tests/dumpkit/test_fallback_cycle_unit.py | 4 +- ...est_no_engine_agreement_prose_1708_unit.py | 14 +- .../test_ext_chain_limit_rebuild_1619_unit.py | 2 +- .../test_ipv6_ext_chain_limit_1604_unit.py | 6 +- .../test_frame_layer_limit_1610_unit.py | 6 +- tests/toolkit/test_pypcapfile_unit.py | 4 +- .../test_tcp_carrier_walk_1606_1617_unit.py | 2 +- tests/utilities/test_config_unit.py | 649 ++++++++++++++++++ tests/utilities/test_logging.py | 25 +- tests/utilities/test_warning_filters.py | 2 +- tests/vendor/test_request_prompt_unit.py | 2 +- tests/vendor/test_user_agent_unit.py | 2 +- 43 files changed, 1680 insertions(+), 299 deletions(-) create mode 100644 docs/source/pcapkit/utilities/config.rst create mode 100644 pcapkit/utilities/config.py create mode 100644 tests/utilities/test_config_unit.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 689dc0e80f..dadabcbae0 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1710,7 +1710,7 @@ jobs: - name: Run the other tests that read docs/ or Markdown timeout-minutes: 5 - run: python -m unittest -v tests.corekit.test_sentinel_exports_unit + run: python -m unittest -v tests.corekit.test_sentinel_exports_unit tests.utilities.test_config_unit # The verdict a shipping workflow asks for: the full suite, fixtures and # all, on one interpreter. It is deliberately not the matrix -- the matrix diff --git a/docs/source/contributing/pep.rst b/docs/source/contributing/pep.rst index f383406c87..a0787ee64c 100644 --- a/docs/source/contributing/pep.rst +++ b/docs/source/contributing/pep.rst @@ -428,9 +428,9 @@ of the frame raw, silently, at a depth that varied with the interpreter and the caller's stack (:issue:`1604`, :issue:`1610`). Recursion stays, with a counted bound: -:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` caps a frame at 64 layers, -and :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` caps one -chain of extension headers at 32. Past either, the rest stays +:data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT` caps a frame at 64 layers by +default, and :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` caps one +chain of extension headers at 32 by default. Past either, the rest stays :class:`~pcapkit.protocols.misc.raw.Raw`, with one :exc:`~pcapkit.utilities.warnings.ProtocolWarning`. Raising :func:`sys.setrecursionlimit` instead was rejected: it is process-wide, and the diff --git a/docs/source/pcapkit/corekit/fields/field.rst b/docs/source/pcapkit/corekit/fields/field.rst index 06561c21f2..d06424efb8 100644 --- a/docs/source/pcapkit/corekit/fields/field.rst +++ b/docs/source/pcapkit/corekit/fields/field.rst @@ -3,6 +3,20 @@ Base Fields .. module:: pcapkit.corekit.fields.field +.. versionremoved:: 1.5.0 + + Moved to :mod:`pcapkit.utilities.config` (:issue:`1726`): + + * ``pcapkit.corekit.fields.field._MAX_ZERO_PAD_LENGTH``, now + :data:`pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`, set with + :envvar:`PCAPKIT_MAX_ZERO_PAD_LENGTH`; + * ``pcapkit.corekit.fields.field._MAX_ZERO_PAD_SHORTFALL``, now + :data:`pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL`, set with + :envvar:`PCAPKIT_MAX_ZERO_PAD_SHORTFALL`; + * ``pcapkit.corekit.fields.field._ZERO_PAD_BUDGET_RATIO``, now + :data:`pcapkit.utilities.config.ZERO_PAD_BUDGET_RATIO`, set with + :envvar:`PCAPKIT_ZERO_PAD_BUDGET_RATIO`. + .. autoclass:: pcapkit.corekit.fields.field.Field :members: :show-inheritance: diff --git a/docs/source/pcapkit/corekit/io.rst b/docs/source/pcapkit/corekit/io.rst index 7f8c0e26e7..f4dde51809 100644 --- a/docs/source/pcapkit/corekit/io.rst +++ b/docs/source/pcapkit/corekit/io.rst @@ -10,6 +10,12 @@ I/O Objects stream, and :class:`~pcapkit.corekit.io.NamedStream`, which gives a stream a ``name``. +.. versionremoved:: 1.5.0 + + ``pcapkit.corekit.io._READ_CHUNK``, now + :data:`pcapkit.utilities.config.READ_CHUNK`, set with + :envvar:`PCAPKIT_READ_CHUNK` (:issue:`1726`). + .. autoclass:: pcapkit.corekit.io.SeekableReader :no-members: :show-inheritance: diff --git a/docs/source/pcapkit/dumpkit/common.rst b/docs/source/pcapkit/dumpkit/common.rst index de7d55dbd5..cdedc58fc6 100644 --- a/docs/source/pcapkit/dumpkit/common.rst +++ b/docs/source/pcapkit/dumpkit/common.rst @@ -7,6 +7,12 @@ Common Utilities :mod:`pcapkit.dumpkit`, chiefly customised hooks for :class:`dictdumper.Dumper` classes. +.. versionremoved:: 1.5.0 + + ``pcapkit.dumpkit.common.FALLBACK_LIMIT``, now + :data:`pcapkit.utilities.config.FALLBACK_LIMIT`, set with + :envvar:`PCAPKIT_FALLBACK_LIMIT` (:issue:`1726`). + .. autofunction:: pcapkit.dumpkit.common.make_dumper .. autoclass:: pcapkit.dumpkit.common.Dumper diff --git a/docs/source/pcapkit/index.rst b/docs/source/pcapkit/index.rst index 9b43bc2647..9afd67cd55 100644 --- a/docs/source/pcapkit/index.rst +++ b/docs/source/pcapkit/index.rst @@ -139,14 +139,6 @@ Environment Variables :data:`pcapkit.utilities.logging.VERBOSE` -.. envvar:: PCAPKIT_HTTP_PROXY - - HTTP proxy address for :mod:`pcapkit.vendor` crawlers. - -.. envvar:: PCAPKIT_HTTPS_PROXY - - HTTPS proxy address for :mod:`pcapkit.vendor` crawlers. - .. envvar:: PCAPKIT_SPHINX If set to ``1``, :mod:`pcapkit` will run with Sphinx additional @@ -155,3 +147,20 @@ Environment Variables .. seealso:: :data:`pcapkit.utilities.logging.SPHINX_TYPE_CHECKING` + +The tunable limits, and the :mod:`pcapkit.vendor` crawlers' +:envvar:`PCAPKIT_VENDOR_RETRY`, :envvar:`PCAPKIT_CI_MODE`, +:envvar:`PCAPKIT_HTTP_PROXY` and :envvar:`PCAPKIT_HTTPS_PROXY`, are read by +:mod:`pcapkit.utilities.config`, which lists each with its default, its +accepted range and its risks. + +.. versionchanged:: 1.5.0 + + :envvar:`PCAPKIT_HTTP_PROXY` and :envvar:`PCAPKIT_HTTPS_PROXY` are documented + with :mod:`pcapkit.utilities.config` rather than here, and are still read on + every request. The tunable limits are new as variables (:issue:`1726`). A + bad value of :envvar:`PCAPKIT_DEVMODE`, :envvar:`PCAPKIT_VERBOSE`, + :envvar:`PCAPKIT_SPHINX`, :envvar:`PCAPKIT_CI_MODE` or + :envvar:`PCAPKIT_VENDOR_RETRY` now warns with + :exc:`~pcapkit.utilities.warnings.ConfigWarning` instead of being taken + silently, or, for :envvar:`PCAPKIT_VENDOR_RETRY`, crashing the import. diff --git a/docs/source/pcapkit/protocols/internet/internet.rst b/docs/source/pcapkit/protocols/internet/internet.rst index d1f610893c..065a106f9e 100644 --- a/docs/source/pcapkit/protocols/internet/internet.rst +++ b/docs/source/pcapkit/protocols/internet/internet.rst @@ -33,7 +33,7 @@ Extension Header Chains ----------------------- A chain of IPv6 extension headers is dissected to at most -:data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` headers in +:data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` headers in a row, whether it follows IPv6 or another layer such as IPv4. What follows is kept as :class:`~pcapkit.protocols.misc.raw.Raw`, with one :exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1604`). The @@ -42,8 +42,12 @@ so it may still find the upper layer past it (:issue:`1609`). So may the ``dpkt`` toolkit, which reads the chain as :mod:`dpkt` decodes it. So does the ``scapy`` engine behind IPv6, where Scapy dissects each extension header itself; behind IPv4, its toolkit steps over them in the octets to find TCP, up to -:data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of them in a -row. The ``pypcapfile`` toolkit walks no extension header chain, so -it finds no upper layer behind one. +:data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` of them in a row. The +``pypcapfile`` toolkit walks no extension header chain, so it finds no upper +layer behind one. -.. autodata:: pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT +.. versionremoved:: 1.5.0 + + ``pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT``, now + :data:`pcapkit.utilities.config.EXTENSION_HEADER_LIMIT`, set with + :envvar:`PCAPKIT_EXTENSION_HEADER_LIMIT` (:issue:`1726`). diff --git a/docs/source/pcapkit/protocols/protocol.rst b/docs/source/pcapkit/protocols/protocol.rst index ad3131dc40..da2dfe91af 100644 --- a/docs/source/pcapkit/protocols/protocol.rst +++ b/docs/source/pcapkit/protocols/protocol.rst @@ -105,7 +105,7 @@ Layers per Frame ---------------- A frame is dissected to at most -:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of its protocol +:data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT` layers of its protocol chain, counted from the first layer the frame record carries, through tunnels, VLAN tags and IPv6 extension headers alike. What follows is kept as :class:`~pcapkit.protocols.misc.raw.Raw`, with one @@ -115,12 +115,16 @@ default engine's TCP reassembly and flow tracing do not read a TCP segment there back out, as they do one the TCP parser rejected (:issue:`1518`). A protocol constructed from its own octets is not counted itself, so the count starts at the first layer it dissects. One chain of extension headers is also bounded on -its own, by :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`. +its own, by :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT`. The ``pyshark``, ``scapy`` and ``pypcapfile`` engines dissect with their own parsers, which do not apply the frame's layer bound (:issue:`1609`). The ``dpkt`` toolkit reads what :mod:`dpkt` decodes, which applies neither bound. -.. autodata:: pcapkit.protocols.protocol.FRAME_LAYER_LIMIT +.. versionremoved:: 1.5.0 + + ``pcapkit.protocols.protocol.FRAME_LAYER_LIMIT``, now + :data:`pcapkit.utilities.config.FRAME_LAYER_LIMIT`, set with + :envvar:`PCAPKIT_FRAME_LAYER_LIMIT` (:issue:`1726`). Data Models ----------- diff --git a/docs/source/pcapkit/toolkit/3rdparty.rst b/docs/source/pcapkit/toolkit/3rdparty.rst index 72fe888984..2057e93252 100644 --- a/docs/source/pcapkit/toolkit/3rdparty.rst +++ b/docs/source/pcapkit/toolkit/3rdparty.rst @@ -182,6 +182,12 @@ cannot be used for it. .. _PyPCAPFile: https://github.com/kisom/pypcapfile +.. versionremoved:: 1.5.0 + + ``pcapkit.toolkit.pypcapfile.IPV6_TCP_WARNED_LIMIT``, now + :data:`pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT`, set with + :envvar:`PCAPKIT_IPV6_TCP_WARNED_LIMIT` (:issue:`1726`). + .. note:: `PyPCAPFile`_ has no IPv6 decoder, so :func:`~pcapkit.toolkit.pypcapfile.ipv6_reassembly` diff --git a/docs/source/pcapkit/utilities/chardet.rst b/docs/source/pcapkit/utilities/chardet.rst index 86e253b162..f57e725d2a 100644 --- a/docs/source/pcapkit/utilities/chardet.rst +++ b/docs/source/pcapkit/utilities/chardet.rst @@ -14,4 +14,10 @@ which is why it lives here rather than beside either of them. .. autofunction:: pcapkit.utilities.chardet.detect -.. autodata:: pcapkit.utilities.chardet.DETECT_CACHE_SIZE +The cache holds :data:`~pcapkit.utilities.config.DETECT_CACHE_SIZE` entries. + +.. versionremoved:: 1.5.0 + + ``pcapkit.utilities.chardet.DETECT_CACHE_SIZE``, now + :data:`pcapkit.utilities.config.DETECT_CACHE_SIZE`, set with + :envvar:`PCAPKIT_DETECT_CACHE_SIZE` (:issue:`1726`). diff --git a/docs/source/pcapkit/utilities/config.rst b/docs/source/pcapkit/utilities/config.rst new file mode 100644 index 0000000000..f034f59bed --- /dev/null +++ b/docs/source/pcapkit/utilities/config.rst @@ -0,0 +1,160 @@ +Configuration +============= + +.. module:: pcapkit.utilities.config + +:mod:`pcapkit.utilities.config` holds every tunable limit of :mod:`pcapkit`, and +the settings of the :mod:`pcapkit.vendor` crawlers (:issue:`1726`). Each limit +is read from a ``PCAPKIT_*`` environment variable once, when the module is +imported: + +* A value that is not an integer, or is below the variable's minimum, warns with + a :exc:`~pcapkit.utilities.warnings.ConfigWarning` and falls back to the + default. +* A limit takes ``-1`` for no limit. That, or a value above the default, is + accepted and warns with a :exc:`~pcapkit.utilities.warnings.ConfigWarning` + naming the risk below. + +Each warning names the variable and the value it was given, e.g. +``PCAPKIT_FALLBACK_LIMIT='-7' is not an integer of at least 0, or -1 for no +limit; using the default, 10000``. It obeys the warning filters, so under +:option:`-W error <-W>` or ``PYTHONWARNINGS=error`` an out-of-range value makes +``import pcapkit`` raise :exc:`~pcapkit.utilities.warnings.ConfigWarning`. + +Readers look each value up as ``config.`` at call time, so patching a name +here reaches every engine and protocol that reads it. The proxy addresses are +read on every call of :func:`~pcapkit.utilities.config.http_proxy` and +:func:`~pcapkit.utilities.config.https_proxy` instead. + +:envvar:`PCAPKIT_DEVMODE`, :envvar:`PCAPKIT_VERBOSE` and :envvar:`PCAPKIT_SPHINX` +are read by :mod:`pcapkit.utilities.logging`, which needs them while it loads +and so cannot import this module. A bad value of one falls back to off, and is +reported with a :exc:`~pcapkit.utilities.warnings.ConfigWarning` once +:mod:`pcapkit.utilities.warnings` has loaded. + +.. versionadded:: 1.5.0 + +Limits +------ + +.. list-table:: + :header-rows: 1 + + * - Variable + - Default + - Accepted + - Risk past the default, or at ``-1`` + * - :envvar:`PCAPKIT_FRAME_LAYER_LIMIT` + - ``64`` + - ``>= 0``, or ``-1`` + - Python's recursion limit: a deep frame may raise :exc:`RecursionError` + (:issue:`1610`) + * - :envvar:`PCAPKIT_EXTENSION_HEADER_LIMIT` + - ``32`` + - ``>= 0``, or ``-1`` + - as above, once :envvar:`PCAPKIT_FRAME_LAYER_LIMIT` is raised too + * - :envvar:`PCAPKIT_IPV6_TCP_WARNED_LIMIT` + - ``32`` + - ``>= 0``, or ``-1`` + - each capture remembered pins one packet in memory + * - :envvar:`PCAPKIT_FALLBACK_LIMIT` + - ``10000`` + - ``>= 0``, or ``-1`` + - a large or endless object graph is dumped without bound + * - :envvar:`PCAPKIT_DETECT_CACHE_SIZE` + - ``1024`` + - ``>= 0``, or ``-1`` + - the cache retains every bytestring it was asked about + * - :envvar:`PCAPKIT_READ_CHUNK` + - ``1048576`` + - ``>= 1``, or ``-1`` + - a bogus record length allocates that much in one read + * - :envvar:`PCAPKIT_MAX_ZERO_PAD_LENGTH` + - ``262144`` + - ``>= 0``, or ``-1`` to lift the ceiling and the running budget + - a hostile capture's declared length is zero-filled in memory + (:issue:`554`) + * - :envvar:`PCAPKIT_MAX_ZERO_PAD_SHORTFALL` + - ``65536`` + - ``>= 0``, or ``-1`` to disable the running budget + - lying lengths amplify into memory without being charged (:issue:`573`) + * - :envvar:`PCAPKIT_ZERO_PAD_BUDGET_RATIO` + - ``16`` + - ``>= 0``, or ``-1`` to lift the running budget + - as above, per octet actually read (:issue:`573`) + +.. envvar:: PCAPKIT_FRAME_LAYER_LIMIT + + Sets :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`. + +.. envvar:: PCAPKIT_EXTENSION_HEADER_LIMIT + + Sets :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT`. + +.. envvar:: PCAPKIT_IPV6_TCP_WARNED_LIMIT + + Sets :data:`~pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT`. + +.. envvar:: PCAPKIT_FALLBACK_LIMIT + + Sets :data:`~pcapkit.utilities.config.FALLBACK_LIMIT`. + +.. envvar:: PCAPKIT_DETECT_CACHE_SIZE + + Sets :data:`~pcapkit.utilities.config.DETECT_CACHE_SIZE`. Read once, when + :mod:`pcapkit.utilities.chardet` is imported. + +.. envvar:: PCAPKIT_READ_CHUNK + + Sets :data:`~pcapkit.utilities.config.READ_CHUNK`. + +.. envvar:: PCAPKIT_MAX_ZERO_PAD_LENGTH + + Sets :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`. + +.. envvar:: PCAPKIT_MAX_ZERO_PAD_SHORTFALL + + Sets :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL`. + +.. envvar:: PCAPKIT_ZERO_PAD_BUDGET_RATIO + + Sets :data:`~pcapkit.utilities.config.ZERO_PAD_BUDGET_RATIO`. + +.. autodata:: pcapkit.utilities.config.FRAME_LAYER_LIMIT +.. autodata:: pcapkit.utilities.config.EXTENSION_HEADER_LIMIT +.. autodata:: pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT +.. autodata:: pcapkit.utilities.config.FALLBACK_LIMIT +.. autodata:: pcapkit.utilities.config.DETECT_CACHE_SIZE +.. autodata:: pcapkit.utilities.config.READ_CHUNK +.. autodata:: pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH +.. autodata:: pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL +.. autodata:: pcapkit.utilities.config.ZERO_PAD_BUDGET_RATIO + +Vendor Crawlers +--------------- + +.. envvar:: PCAPKIT_VENDOR_RETRY + + Sets :data:`~pcapkit.utilities.config.VENDOR_RETRY`: an integer ``>= 1``, + default ``5``. + +.. envvar:: PCAPKIT_CI_MODE + + Sets :data:`~pcapkit.utilities.config.CI_MODE`: ``1``/``yes``/``true``/``on`` + or ``0``/``no``/``false``/``off``, default off. + +.. envvar:: PCAPKIT_HTTP_PROXY + + The HTTP proxy address for the :mod:`pcapkit.vendor` crawlers, read by + :func:`~pcapkit.utilities.config.http_proxy` on every call. Unset by default. + +.. envvar:: PCAPKIT_HTTPS_PROXY + + The HTTPS proxy address for the :mod:`pcapkit.vendor` crawlers, read by + :func:`~pcapkit.utilities.config.https_proxy` on every call. Unset by + default. + +.. autodata:: pcapkit.utilities.config.VENDOR_RETRY +.. autodata:: pcapkit.utilities.config.CI_MODE +.. autofunction:: pcapkit.utilities.config.http_proxy +.. autofunction:: pcapkit.utilities.config.https_proxy diff --git a/docs/source/pcapkit/utilities/index.rst b/docs/source/pcapkit/utilities/index.rst index 7d70aee097..26290bd47d 100644 --- a/docs/source/pcapkit/utilities/index.rst +++ b/docs/source/pcapkit/utilities/index.rst @@ -15,6 +15,7 @@ layer, and wrappers around third-party libraries. It imports nothing from exceptions warnings logging + config chardet Version Compatibility diff --git a/docs/source/pcapkit/utilities/warnings.rst b/docs/source/pcapkit/utilities/warnings.rst index 65377d4cdf..6d613ce05c 100644 --- a/docs/source/pcapkit/utilities/warnings.rst +++ b/docs/source/pcapkit/utilities/warnings.rst @@ -166,6 +166,10 @@ configured separately, through the :mod:`logging` module:: :no-members: :show-inheritance: +.. autoexception:: pcapkit.utilities.warnings.ConfigWarning + :no-members: + :show-inheritance: + :exc:`ResourceWarning` Category ------------------------------- diff --git a/docs/source/pcapkit/vendor/default.rst b/docs/source/pcapkit/vendor/default.rst index 3b13a26b7b..c9f5c9bc59 100644 --- a/docs/source/pcapkit/vendor/default.rst +++ b/docs/source/pcapkit/vendor/default.rst @@ -6,6 +6,17 @@ Base Crawler :mod:`pcapkit.vendor.default` contains :class:`~pcapkit.vendor.default.Vendor` only, which is the base meta class for all vendor crawlers. +.. versionremoved:: 1.5.0 + + Moved to :mod:`pcapkit.utilities.config` (:issue:`1726`): + + * ``pcapkit.vendor.default.MAX_RETRY``, now + :data:`pcapkit.utilities.config.VENDOR_RETRY`, set with + :envvar:`PCAPKIT_VENDOR_RETRY`; + * ``pcapkit.vendor.default.CI_MODE``, now + :data:`pcapkit.utilities.config.CI_MODE`, set with + :envvar:`PCAPKIT_CI_MODE`. + .. autoclass:: pcapkit.vendor.default.Vendor :no-members: :show-inheritance: diff --git a/pcapkit/corekit/fields/field.py b/pcapkit/corekit/fields/field.py index cf0ae09119..dda005e115 100644 --- a/pcapkit/corekit/fields/field.py +++ b/pcapkit/corekit/fields/field.py @@ -1,5 +1,26 @@ # -*- coding: utf-8 -*- -"""base field class""" +"""Base Fields +================ + +.. module:: pcapkit.corekit.fields.field + +:mod:`pcapkit.corekit.fields.field` contains the base field classes. + +.. versionremoved:: 1.5.0 + + Moved to :mod:`pcapkit.utilities.config` (:issue:`1726`): + + * ``pcapkit.corekit.fields.field._MAX_ZERO_PAD_LENGTH``, now + :data:`pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`, set with + :envvar:`PCAPKIT_MAX_ZERO_PAD_LENGTH`; + * ``pcapkit.corekit.fields.field._MAX_ZERO_PAD_SHORTFALL``, now + :data:`pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL`, set with + :envvar:`PCAPKIT_MAX_ZERO_PAD_SHORTFALL`; + * ``pcapkit.corekit.fields.field._ZERO_PAD_BUDGET_RATIO``, now + :data:`pcapkit.utilities.config.ZERO_PAD_BUDGET_RATIO`, set with + :envvar:`PCAPKIT_ZERO_PAD_BUDGET_RATIO`. + +""" import abc import contextlib @@ -9,6 +30,7 @@ from typing import TYPE_CHECKING, Generic, TypeVar, cast from pcapkit.corekit.sentinels import NO_VALUE, NoValueType # pylint: disable=unused-import +from pcapkit.utilities import config from pcapkit.utilities.exceptions import FieldValueError, NoDefaultValue, ProtocolError __all__ = ['NO_VALUE', 'Field'] @@ -22,99 +44,9 @@ _T = TypeVar('_T') -#: int: Ceiling on the zero-padding :meth:`FieldBase.unpack` will still perform -#: for a field whose declared length outruns its buffer. -#: -#: This is libpcap's own ``MAXIMUM_SNAPLEN`` -- the point past which libpcap -#: itself treats a capture's declared snapshot length as corrupt or -#: byte-order-swapped rather than real -- and it is the same figure this -#: package's own PCAP writer defaults ``snaplen`` to -#: (:meth:`pcapkit.protocols.misc.pcap.header.Header.make`). No single field -#: within one captured packet is legitimately larger than the largest packet -#: libpcap itself is willing to believe, so nothing this library parses -#: should ever declare a length past it. -#: -#: The bound is deliberately *not* "any declared length beyond what the -#: buffer holds": :meth:`ListField.unpack ` and :meth:`OptionField.unpack ` depend on reading a short, sometimes -#: empty, tail past a truncated area and having it decode as zero -- that is -#: how an over-long ``ihl``, or a capture cut short by the snapshot length, -#: reads as end-of-option-list or ``Pad1`` instead of wedging or raising (see -#: :issue:`431`). Every such read is of a fixed-width, few-octet field, always -#: far under this ceiling, so it is untouched; only a length past it -- which -#: no fixed-width field ever legitimately is -- gets refused. -#: -#: A ceiling on one field says nothing about how many fields a parse may pad, -#: which is what :data:`_MAX_ZERO_PAD_SHORTFALL` and -#: :data:`_ZERO_PAD_BUDGET_RATIO` below are for. See :issue:`573`. -_MAX_ZERO_PAD_LENGTH = 0x40_000 - -#: int: Shortfall :meth:`FieldBase.unpack` will always zero-pad for, whatever -#: else a parse has already padded. -#: -#: :data:`_MAX_ZERO_PAD_LENGTH` bounds each field on its own, and a packet holds -#: many fields, so it leaves the *sum* unbounded: a declared length just under the -#: ceiling is honoured however often it is declared. Measured on this tree, 200 -#: minimal PCAP-NG Decryption Secrets Blocks -- 4,800 wire octets, each block -#: declaring ``secrets_length`` of 262,142 against two supplied octets through -#: ``UnknownSecrets.data`` (``pcapkit/protocols/schema/misc/pcapng.py``) -- -#: retained 50.0 MiB, an amplification of 10,922x per block. Every individual -#: field was under the ceiling, so nothing refused any of them (:issue:`573`). -#: -#: The sum therefore wants a budget, and this is the figure that makes one -#: *safe*. A budget on its own is not: a capture cut short by its snapshot length -#: pads legitimately and must keep parsing (:issue:`431`, and the reasoning that -#: declined :issue:`554`), and it pads far more than it reads, so any running -#: budget tight enough to matter starts refusing real captures. Worse, it refuses -#: them *sometimes* -- measured on this tree with a running budget alone, the -#: same legitimate 54-octet frame parsed to one result on 37 of 40 calls and to -#: another on calls 26, 33 and 39, because whether it fit depended on what had -#: been parsed before it. A guard whose answer moves with history is not a guard. -#: -#: 65,536 is what removes that. It is the whole span of a 16-bit wire length -#: field -- which is how an IP header, an IPv6 payload, a TCP or IPv4 option and -#: a PCAP-NG option all declare their size -- so no shortfall that one of those -#: can produce is subject to the budget at all, and every one of them is padded -#: unconditionally. Measured: the largest legitimate single -#: shortfall found anywhere was 65,495 octets, from a snapshot-truncated -#: offload-sized frame (100 frames at ``incl_len`` 54 declaring an IPv4 total -#: length of 65,535, padding 6,549,500 octets from 7,024 read), and 64,750 from a -#: truncated PCAP-NG option. Both sit under this figure, and so does anything -#: else a 16-bit field can ask for. -#: -#: What is left above it is the band a *32-bit* wire length reaches -- -#: PCAP-NG's own block and secrets lengths, which is where :issue:`573`'s -#: amplification lives -- and legitimately that is a once-per-file event, since -#: only the last block of a truncated capture is cut short. So the band gets the -#: running budget below, whose one-off term already covers any single such event -#: outright. -_MAX_ZERO_PAD_SHORTFALL = 0x10_000 - -#: int: Zero padding *past* :data:`_MAX_ZERO_PAD_SHORTFALL` that -#: :meth:`FieldBase.unpack` will synthesise in total for every octet a parse has -#: actually been given, over and above the one-off -#: :data:`_MAX_ZERO_PAD_LENGTH` allowance. -#: -#: A shortfall in this band -- past 65,536 octets and so past anything a 16-bit -#: wire length can declare, but within :data:`_MAX_ZERO_PAD_LENGTH` -- is a -#: PCAP-NG block or secrets length, a 32-bit figure. One of those, on its own, is -#: legitimate: it is what the final block of a capture truncated at EOF looks -#: like, and the ``_MAX_ZERO_PAD_LENGTH`` term of the allowance covers it whole, -#: from a cold start, because such a shortfall cannot exceed that ceiling and -#: still be padded at all. -#: -#: *Repeating* one is not legitimate, and that is what 16 is chosen to catch. -#: Truncation cuts the end of a file, so a capture has one short block, not two -#: hundred; :issue:`573`'s shape has two hundred because they are declared -#: rather than cut. 16 octets of further allowance per octet genuinely read -#: leaves any real file an allowance orders of magnitude past the one event it -#: can want, while bounding the sum for a file whose blocks all lie. -_ZERO_PAD_BUDGET_RATIO = 0x10 - #: ContextVar[Optional[list[int]]]: Running ``[octets supplied, octets #: synthesised]`` ledger for :meth:`FieldBase.unpack`, against which -#: :data:`_ZERO_PAD_BUDGET_RATIO` is enforced. +#: :data:`~pcapkit.utilities.config.ZERO_PAD_BUDGET_RATIO` is enforced. #: #: A :class:`~contextvars.ContextVar` rather than a plain module global so that #: a thread parsing one capture cannot spend the budget of a thread parsing @@ -140,8 +72,8 @@ #: calls :func:`_zero_pad_budget` -- so the bound as shipped is over everything a #: context has ever parsed, not over one ``Extractor`` run. It is still a bound, #: and still proportionate: total padding past -#: :data:`_MAX_ZERO_PAD_SHORTFALL` stays within -#: ``_MAX_ZERO_PAD_LENGTH + _ZERO_PAD_BUDGET_RATIO`` times the octets that +#: :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL` stays within +#: ``MAX_ZERO_PAD_LENGTH + ZERO_PAD_BUDGET_RATIO`` times the octets that #: context has genuinely been given, so a long-running service that has read a #: great deal has earned proportionately more, rather than banking an unlimited #: allowance. Scoping it per file would make the bound tighter and is a one-line @@ -149,7 +81,7 @@ #: layer has something to call. #: #: Cumulative state is what makes an answer depend on history, so note what is -#: *not* weighed against it: a shortfall within :data:`_MAX_ZERO_PAD_SHORTFALL` +#: *not* weighed against it: a shortfall within :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL` #: neither consults this ledger nor is charged to it, which is what keeps every #: legitimate short read answering the same however much preceded it. Only the #: 32-bit band above reads and writes ``[1]``. @@ -462,11 +394,11 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> '_T': Raises: FieldValueError: If ``buffer`` holds fewer octets than :attr:`length` declares, and either ``length`` is past - :data:`_MAX_ZERO_PAD_LENGTH`, or the shortfall is past - :data:`_MAX_ZERO_PAD_SHORTFALL` and takes this parse's total of - such shortfalls past what :data:`_ZERO_PAD_BUDGET_RATIO` allows + :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`, or the shortfall is past + :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL` and takes this parse's total of + such shortfalls past what :data:`~pcapkit.utilities.config.ZERO_PAD_BUDGET_RATIO` allows for the octets it has actually been given. A shortfall within - :data:`_MAX_ZERO_PAD_SHORTFALL` is always padded and never + :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL` is always padded and never raises. """ @@ -483,20 +415,21 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> '_T': # field from a value it just read off the wire (e.g. ``DecryptionSecretsBlock``'s # ``secrets_data: BytesField(length=lambda pkt: pkt['__length__'])``) -- # and is thus attacker-controlled: a corrupt or hostile capture can - # declare an arbitrarily large one. Past :data:`_MAX_ZERO_PAD_LENGTH`, a + # declare an arbitrarily large one. Past :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`, a # length short of what ``buffer`` holds is provably bogus: the padding # cannot recover data that was never in the buffer, only zero-fill for # it, and no field this large is legitimate to begin with. Honouring it # would allocate and zero-fill up to ``length`` octets on nothing but # the packet's own say-so. C.f. #554. - if length > _MAX_ZERO_PAD_LENGTH and len(buffer) < length: + ceiling = config.MAX_ZERO_PAD_LENGTH + if ceiling != -1 and length > ceiling and len(buffer) < length: raise FieldValueError( f'Field {self.name} declares a length of {length} octet(s), ' f'but only {len(buffer)} octet(s) are available.' ) # NOTE: The ceiling above bounds one field; this bounds their sum. A - # declared length just under :data:`_MAX_ZERO_PAD_LENGTH` passes it every + # declared length just under :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH` passes it every # time it is declared, so 200 of them amplify 4,800 wire octets into # 50.0 MiB of retained zeros with the ceiling doing exactly what it was # written to do. C.f. #573. @@ -536,7 +469,7 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> '_T': _zero_pad_ledger.set(ledger) ledger[0] += supplied - # A shortfall no larger than :data:`_MAX_ZERO_PAD_SHORTFALL` is padded + # A shortfall no larger than :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL` is padded # without consulting the budget, and is not charged to it either. That is # the load-bearing half of this fix rather than a concession in it: it is # what keeps the answer a function of *this* read rather than of @@ -551,13 +484,16 @@ def unpack(self, buffer: 'bytes | IO[bytes]', packet: 'dict[str, Any]') -> '_T': # band; charging that to the same ledger would exhaust it within a few # frames and refuse the next large shortfall -- reintroducing exactly the # history-dependence the band exists to remove. - if padding > _MAX_ZERO_PAD_SHORTFALL: - allowance = _MAX_ZERO_PAD_LENGTH + _ZERO_PAD_BUDGET_RATIO * ledger[0] - if ledger[1] + padding > allowance: + shortfall, ratio = config.MAX_ZERO_PAD_SHORTFALL, config.ZERO_PAD_BUDGET_RATIO + if shortfall != -1 and padding > shortfall: + # NOTE: ``-1`` for either term of the allowance lifts the budget. + unbounded = ceiling == -1 or ratio == -1 + allowance = 0 if unbounded else ceiling + ratio * ledger[0] + if not unbounded and ledger[1] + padding > allowance: raise FieldValueError( f'Field {self.name} would zero-pad {padding} octet(s), ' f'taking this parse to {ledger[1] + padding} octet(s) of ' - f'padding past {_MAX_ZERO_PAD_SHORTFALL} octet(s) against ' + f'padding past {shortfall} octet(s) against ' f'{ledger[0]} octet(s) actually read, past the {allowance} ' f'octet(s) allowed.' ) diff --git a/pcapkit/corekit/io.py b/pcapkit/corekit/io.py index f77db6b37f..1a7bf9e492 100644 --- a/pcapkit/corekit/io.py +++ b/pcapkit/corekit/io.py @@ -11,11 +11,18 @@ seekable stream, and :class:`~pcapkit.corekit.io.NamedStream`, which gives a stream a ``name``. +.. versionremoved:: 1.5.0 + + ``pcapkit.corekit.io._READ_CHUNK``, now + :data:`pcapkit.utilities.config.READ_CHUNK`, set with + :envvar:`PCAPKIT_READ_CHUNK` (:issue:`1726`). + """ import io import tempfile from typing import TYPE_CHECKING, cast +from pcapkit.utilities import config from pcapkit.utilities.exceptions import (SeekError, TruncateError, UnsupportedCall, UnsupportedOperation, stacklevel) from pcapkit.utilities.warnings import SeekWarning, warn @@ -28,10 +35,6 @@ __all__ = ['SeekableReader', 'PeekableStream', 'NamedStream'] -#: Most octets :class:`SeekableReader` asks of its stream in one read. A read -#: allocates what it asks for up front, so a record's bogus length is read in pieces. -_READ_CHUNK = 1 << 20 - class SeekableReader(io.BufferedReader): """Seekable buffered reader. @@ -179,8 +182,8 @@ def _fill(self, size: 'int', minimum: 'Optional[int]' = None, /, *, reads until ``minimum`` octets are in hand. Past that it does not wait: ``read1``, where the stream has it, returns what is already there. A non-blocking stream's :obj:`None` stops the reading. No one read asks for - more than ``_READ_CHUNK`` octets, so a bogus length read from a record allocates - only what the stream actually holds. + more than :data:`~pcapkit.utilities.config.READ_CHUNK` octets, so a bogus + length read from a record allocates only what the stream actually holds. """ if size < 0: @@ -191,10 +194,11 @@ def _fill(self, size: 'int', minimum: 'Optional[int]' = None, /, *, minimum = size if minimum is None else minimum read = getattr(self._stream, 'read1', self._stream.read) + limit = config.READ_CHUNK chunks = [] # type: list[bytes] count = 0 while True: - chunk = read(min(size - count, _READ_CHUNK)) + chunk = read(size - count if limit == -1 else min(size - count, limit)) if not chunk: break chunks.append(chunk) diff --git a/pcapkit/dumpkit/common.py b/pcapkit/dumpkit/common.py index 8e8f93decf..59a1d1637f 100644 --- a/pcapkit/dumpkit/common.py +++ b/pcapkit/dumpkit/common.py @@ -9,6 +9,12 @@ generally the customised hooks for :class:`dictdumper.Dumper` classes. +.. versionremoved:: 1.5.0 + + ``pcapkit.dumpkit.common.FALLBACK_LIMIT``, now + :data:`pcapkit.utilities.config.FALLBACK_LIMIT`, set with + :envvar:`PCAPKIT_FALLBACK_LIMIT` (:issue:`1726`). + """ import collections import datetime @@ -33,6 +39,7 @@ from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict from pcapkit.corekit.sentinels import NoValueType, NullType from pcapkit.protocols.schema.schema import Schema +from pcapkit.utilities import config from pcapkit.utilities.exceptions import UnsupportedCall from pcapkit.utilities.logging import get_logger @@ -271,12 +278,6 @@ def _iter_slots(value: 'Any') -> 'Iterator[str]': yield name -#: Most objects :meth:`DictDumper._append_fallback` expands for one top-level -#: object, counting an object once per time it is met. No fallback object in the -#: ``foundation``, ``dumpkit`` or ``interface`` test legs expands into more than 7. -FALLBACK_LIMIT = 10_000 - - def _object_name(value: 'Any') -> 'str': """Name a class, module or routine for dumping, in place of its attributes. @@ -728,7 +729,7 @@ def _append_fallback(self, value: 'Any', file: 'TextIO') -> 'None': Raises: UnsupportedCall: If a top-level fallback object expands into - more than :data:`FALLBACK_LIMIT` objects. + more than :data:`~pcapkit.utilities.config.FALLBACK_LIMIT` objects. """ if isinstance(value, (type, types.ModuleType)) or inspect.isroutine(value): @@ -753,9 +754,10 @@ def _append_fallback(self, value: 'Any', file: 'TextIO') -> 'None': self._append_fallback_value(marker, file) return self._fallback_count += 1 - if self._fallback_count > FALLBACK_LIMIT: + limit = config.FALLBACK_LIMIT + if limit != -1 and self._fallback_count > limit: raise UnsupportedCall(f'{type(self).__name__}: fallback object expands into more ' - f'than {FALLBACK_LIMIT} objects; stopped at ' + f'than {limit} objects; stopped at ' f'{_object_name(type(value))}') path.add(key) try: diff --git a/pcapkit/protocols/internet/internet.py b/pcapkit/protocols/internet/internet.py index 9d63d99970..d3a04cd097 100644 --- a/pcapkit/protocols/internet/internet.py +++ b/pcapkit/protocols/internet/internet.py @@ -10,6 +10,12 @@ :class:`~pcapkit.protocols.internet.ipsec.IPsec`, :class:`~pcapkit.protocols.internet.ipv4.IPv4`, :class:`~pcapkit.protocols.internet.ipv6.IPv6`, :class:`~pcapkit.protocols.internet.ipx.IPX`, and etc. +.. versionremoved:: 1.5.0 + + ``pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT``, now + :data:`pcapkit.utilities.config.EXTENSION_HEADER_LIMIT`, set with + :envvar:`PCAPKIT_EXTENSION_HEADER_LIMIT` (:issue:`1726`). + """ import collections import contextvars @@ -23,6 +29,7 @@ from pcapkit.corekit.protochain import ProtoChain from pcapkit.protocols.protocol import _PT, _ST, ProtocolBase, _rebuilding from pcapkit.protocols.schema.schema import keep_short_read, replay_short_read +from pcapkit.utilities import config from pcapkit.utilities.exceptions import stacklevel from pcapkit.utilities.warnings import ProtocolWarning, RegistryWarning, warn @@ -35,32 +42,10 @@ __all__ = ['Internet'] -#: Most IPv6 extension headers dissected in one chain, i.e. one after another -#: with no other layer between them (:issue:`1604`). The walk stops there, and -#: what follows the last of them is kept as -#: :class:`~pcapkit.protocols.misc.raw.Raw`, with one -#: :exc:`~pcapkit.utilities.warnings.ProtocolWarning`. That holds whether the -#: chain follows IPv6, which walks it in a loop -#: (:meth:`IPv6._decode_next_layer -#: `), or another -#: layer such as IPv4, where each header dissects the next one itself -#: (:meth:`Internet._import_next_layer`). The toolkit adapters that walk a -#: chain themselves stop at the same depth. -#: -#: :rfc:`8200#section-4.1` allows each extension header at most once, and the -#: Destination Options header at most twice, so no compliant chain gets close. -#: Without the bound, a chain dissected header by header stops wherever -#: Python's recursion limit falls, which depends on the interpreter. -#: -#: Rebuilding a frame with :meth:`~pcapkit.protocols.protocol.ProtocolBase.from_data` -#: stops where parsing it did, and warns once too (:issue:`1619`), c.f. -#: :meth:`Internet._make_payload`. -EXTENSION_HEADER_LIMIT = 32 - #: contextvars.ContextVar[int]: Where the payload :meth:`Internet._make_payload` #: rebuilds sits in a chain of IPv6 extension headers, should it be one: one past #: the layer it is the payload of, as :meth:`Internet._next_exthdr_depth` places -#: it while parsing, c.f. :data:`EXTENSION_HEADER_LIMIT`. +#: it while parsing, c.f. :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT`. _exthdr_rebuilt = contextvars.ContextVar('_exthdr_rebuilt', default=0) @@ -154,7 +139,7 @@ class Internet(ProtocolBase[_PT, _ST], Generic[_PT, _ST]): # pylint: disable=ab #: int: How many IPv6 extension headers in a row the next layer follows: #: this layer's place in such a chain, or how many headers #: :class:`~pcapkit.protocols.internet.ipv6.IPv6` walked, else ``0``, c.f. - #: :data:`EXTENSION_HEADER_LIMIT` and :meth:`_next_exthdr_depth`. + #: :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` and :meth:`_next_exthdr_depth`. _exthdr_depth = 0 ########################################################################## @@ -239,7 +224,7 @@ def __post_init__(self, file: 'Optional[IO[bytes] | bytes]' = None, _exthdr_depth (int): Where this layer is in a chain of IPv6 extension headers, counting from ``1``, or ``0`` if it is not in one. Handed over by :meth:`_import_next_layer` while - parsing, c.f. :data:`EXTENSION_HEADER_LIMIT`; a layer + parsing, c.f. :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT`; a layer constructed instead, as :meth:`from_data` rebuilds one, takes it from :meth:`_rebuilt_exthdr_depth` (:issue:`1619`). **kwargs: Arbitrary keyword arguments. @@ -300,7 +285,7 @@ def _next_exthdr_depth(self, proto: 'Optional[int]', extension: 'bool' = False) One more than :attr:`_exthdr_depth` if ``proto`` names an IPv6 extension header, or ``0`` if it names none, so that another layer between two headers starts a new chain. Past - :data:`EXTENSION_HEADER_LIMIT`, the next layer is left raw. + :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT`, the next layer is left raw. """ if extension: @@ -340,7 +325,7 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc `, but the payload is built at its place in its chain of IPv6 extension headers, one past this layer's, since it parses its own octets again. So rebuilding a frame - stops at :data:`EXTENSION_HEADER_LIMIT` where parsing it did, rather + stops at :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` where parsing it did, rather than that many headers past each header of the chain, and warns of the limit once, as :meth:`_next_layer_class` describes (:issue:`1619`). @@ -378,14 +363,14 @@ def _next_layer_class(self, proto: 'Optional[int]', file_: 'bytes', length: 'int :class:`~pcapkit.protocols.misc.null.NoPayload` if there is nothing to parse; :class:`~pcapkit.protocols.misc.raw.Raw` if the parse stops at this layer, or the next layer is an IPv6 extension - header past :data:`EXTENSION_HEADER_LIMIT` in a row; otherwise the + header past :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` in a row; otherwise the class registered for ``proto``. With it, the keywords only that next layer is constructed with: the warning given, as ``error``, or the next layer's place in its chain, as ``_exthdr_depth``. Warns: ProtocolWarning: If the next layer is past - :data:`EXTENSION_HEADER_LIMIT` (:issue:`1604`), unless this + :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` (:issue:`1604`), unless this layer is inside a payload being rebuilt (:issue:`1619`). """ @@ -397,15 +382,16 @@ class registered for ``proto``. With it, the keywords only that next return Raw, {} depth = self._next_exthdr_depth(proto, extension) - if depth > EXTENSION_HEADER_LIMIT: + limit = config.EXTENSION_HEADER_LIMIT + if limit != -1 and depth > limit: from pcapkit.protocols.misc.raw import Raw # isort: skip # pylint: disable=import-outside-toplevel message = (f'{self.alias}: a chain of IPv6 extension headers reached the limit of ' - f'{EXTENSION_HEADER_LIMIT} headers; the rest of the packet, from Next ' + f'{limit} headers; the rest of the packet, from Next ' f'Header {int(cast("int", proto))}, is left undissected') # NOTE: Inside a payload being rebuilt, the layer it is rebuilt for # warns of the same octets once its own parse gets here, as for - # :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`. + # :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`. if not _rebuilding.get(): warn(message, ProtocolWarning, stacklevel=stacklevel()) return Raw, {'error': message} @@ -483,7 +469,7 @@ def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *, # Instance of next layer. Notes: - An IPv6 extension header past :data:`EXTENSION_HEADER_LIMIT` in a + An IPv6 extension header past :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` in a row is not dissected: it and the rest of the packet are kept as :class:`~pcapkit.protocols.misc.raw.Raw`, with one :exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1604`). diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 44d343457d..c3a8093848 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -36,11 +36,11 @@ from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict from pcapkit.corekit.protochain import ProtoChain from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6 -from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT from pcapkit.protocols.internet.ip import IP -from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT, _frame_depth +from pcapkit.protocols.protocol import _frame_depth from pcapkit.protocols.schema.internet.ipv6 import IPv6 as Schema_IPv6 from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length, jumbogram_error +from pcapkit.utilities import config from pcapkit.utilities.exceptions import ProtocolError if TYPE_CHECKING: @@ -564,18 +564,18 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, break # NOTE: At most :data:`EXTENSION_HEADER_LIMIT - # ` + # ` # headers are walked, as at most that many are dissected after any # other layer. This header and the rest are then the upper layer, # which :meth:`_import_next_layer` keeps raw, with one warning # (:issue:`1604`). - if len(_protos) >= EXTENSION_HEADER_LIMIT: + if config.EXTENSION_HEADER_LIMIT != -1 and len(_protos) >= config.EXTENSION_HEADER_LIMIT: break # NOTE: Likewise past :data:`FRAME_LAYER_LIMIT - # ` layers of the + # ` layers of the # frame (:issue:`1610`), since each header walked is one of them. - if depth + len(_protos) >= FRAME_LAYER_LIMIT: + if config.FRAME_LAYER_LIMIT != -1 and depth + len(_protos) >= config.FRAME_LAYER_LIMIT: break # # directly break when No Next Header occurs @@ -743,7 +743,7 @@ def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *, # the full distinction. The upper layer past a walk cut at :data:`EXTENSION_HEADER_LIMIT - ` is the + ` is the next extension header, and is kept raw with one :exc:`~pcapkit.utilities.warnings.ProtocolWarning`, as in :meth:`Internet._import_next_layer diff --git a/pcapkit/protocols/protocol.py b/pcapkit/protocols/protocol.py index 4313511b9f..ff0cc1efaf 100644 --- a/pcapkit/protocols/protocol.py +++ b/pcapkit/protocols/protocol.py @@ -10,6 +10,12 @@ an abstract base class for all protocol family, with pre-defined utility arguments and methods of specified protocols. +.. versionremoved:: 1.5.0 + + ``pcapkit.protocols.protocol.FRAME_LAYER_LIMIT``, now + :data:`pcapkit.utilities.config.FRAME_LAYER_LIMIT`, set with + :envvar:`PCAPKIT_FRAME_LAYER_LIMIT` (:issue:`1726`). + """ import abc import ast @@ -47,6 +53,7 @@ from pcapkit.protocols.data.protocol import Packet as Data_Packet from pcapkit.protocols.schema.misc.raw import Raw as Schema_Raw from pcapkit.protocols.schema.schema import Schema +from pcapkit.utilities import config from pcapkit.utilities.chardet import detect from pcapkit.utilities.compat import cached_property, get_annotations from pcapkit.utilities.exceptions import (BaseError, ProtocolError, ProtocolNotFound, @@ -396,34 +403,14 @@ def _check_construction_keywords(cls: 'type', kwargs: 'dict[str, Any]', UnknownFieldWarning) -#: Most layers dissected in one frame (:issue:`1610`). A layer is one entry of -#: the frame's protocol chain, counted from the first layer the frame record -#: carries, usually its link layer; a protocol constructed from its own octets -#: is not counted itself, so the count starts at the first layer it dissects. -#: IPv6 extension headers are layers like any other, whether -#: :class:`~pcapkit.protocols.internet.ipv6.IPv6` walks them or each dissects -#: the next. What follows the last layer is kept as -#: :class:`~pcapkit.protocols.misc.raw.Raw`, with one -#: :exc:`~pcapkit.utilities.warnings.ProtocolWarning`, unless it would have been -#: left raw anyway or there is nothing left. -#: -#: Each layer dissects the next from inside its own parse, so without the bound -#: a deep enough frame, such as IP in IP or stacked 802.1Q tags, stops wherever -#: Python's recursion limit falls, which depends on the interpreter and on how -#: deep the caller's own stack is. A tunnel does not restart the count, and nor -#: does the end of a chain of extension headers, which -#: :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` bounds on -#: its own. -FRAME_LAYER_LIMIT = 64 - #: contextvars.ContextVar[int]: How many layers of the frame being dissected #: come before the next one :meth:`ProtocolBase._parse_next_layer` parses, c.f. -#: :data:`FRAME_LAYER_LIMIT`. +#: :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`. _frame_depth = contextvars.ContextVar('_frame_depth', default=0) #: contextvars.ContextVar[bool]: Whether a payload is being rebuilt by #: :meth:`ProtocolBase._make_payload`, so that the layer it is rebuilt for -#: parses the same octets again, c.f. :data:`FRAME_LAYER_LIMIT`. +#: parses the same octets again, c.f. :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`. _rebuilding = contextvars.ContextVar('_rebuilding', default=False) @@ -574,7 +561,7 @@ class ProtocolBase(Generic[_PT, _ST], metaclass=ProtocolMeta): #: ``object.__new__(SomeProtocol)``) sees, so reading it is always safe. _exctx: 'Optional[ContextRegistry]' = None - #: Whether this layer is past :data:`FRAME_LAYER_LIMIT`, and so was kept as + #: Whether this layer is past :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`, and so was kept as #: :class:`~pcapkit.protocols.misc.raw.Raw` by :meth:`_parse_next_layer` #: without being parsed. Such a layer carries an ``error`` as one that #: failed to parse does, so this is what tells the two apart: what follows @@ -2065,7 +2052,7 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc The payload is built one layer further into the frame than this protocol, as :meth:`_parse_next_layer` counts it, since each layer built parses its own octets again. So rebuilding a frame stops at - :data:`FRAME_LAYER_LIMIT` where parsing it did, rather than that many + :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT` where parsing it did, rather than that many layers below each layer of it (:issue:`1610`). Nor does the payload's parse warn of the limit: this protocol's own parse goes over the same octets, so the rebuilt frame warns of it once, as the parsed one did. @@ -2344,7 +2331,7 @@ def _parse_next_layer(klass: 'Type[ProtocolBase]', file_: 'bytes', length: 'int' Returns: Instance of ``klass``, or of :class:`~pcapkit.protocols.misc.raw.Raw` when fewer octets were captured than its header needs, or when it is - past :data:`FRAME_LAYER_LIMIT`. + past :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`. Note: :meth:`Schema.unpack ` @@ -2373,24 +2360,25 @@ def _parse_next_layer(klass: 'Type[ProtocolBase]', file_: 'bytes', length: 'int' still warns. A layer captured at least as long as its hint cannot be replaced, and is parsed holding nothing, even inside one that holds. - A layer past :data:`FRAME_LAYER_LIMIT` in its frame is not parsed: + A layer past :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT` in its frame is not parsed: it and the rest of the frame are kept as :class:`~pcapkit.protocols.misc.raw.Raw`, marked :attr:`_past_layer_limit`, with one :exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1610`). Warns: - ProtocolWarning: If the next layer is past :data:`FRAME_LAYER_LIMIT`. + ProtocolWarning: If the next layer is past :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`. """ depth = _frame_depth.get() + 1 - if depth > FRAME_LAYER_LIMIT: + limit = config.FRAME_LAYER_LIMIT + if limit != -1 and depth > limit: from pcapkit.protocols.misc.null import NoPayload # isort: skip # pylint: disable=import-outside-toplevel from pcapkit.protocols.misc.raw import Raw # isort: skip # pylint: disable=import-outside-toplevel # NOTE: Nothing is lost if the next layer is raw already, or empty. if klass is not Raw and klass is not NoPayload: - message = (f'the frame reached the limit of {FRAME_LAYER_LIMIT} layers; the rest of it, ' + message = (f'the frame reached the limit of {limit} layers; the rest of it, ' f'from {klass.__name__} on, is left undissected') # NOTE: Inside a payload being rebuilt, the layer it is rebuilt # for warns of the same octets once its own parse gets here. diff --git a/pcapkit/toolkit/pcap.py b/pcapkit/toolkit/pcap.py index 54b3900e30..400ecf6d60 100644 --- a/pcapkit/toolkit/pcap.py +++ b/pcapkit/toolkit/pcap.py @@ -119,7 +119,7 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]': :class:`~pcapkit.protocols.misc.raw.Raw` the TCP parser rejected, of protocol TCP, from a datagram that is not a later fragment, with at least those 20 octets captured; else :data:`None`. A segment past - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` is kept raw + :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT` is kept raw without being parsed, not rejected, so it is not read (:issue:`1610`). Note: @@ -156,7 +156,7 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]': raw = ip.payload # NOTE: a Raw with no error is one the caller asked for, by stopping the # dissection at a ``layer`` or ``protocol``, not a segment TCP rejected. - # Nor is one past FRAME_LAYER_LIMIT, which carries an error all the same: + # Nor is one past config.FRAME_LAYER_LIMIT, which carries an error all the same: # the frame is not dissected past it, so its TCP is not read (#1610). if (not isinstance(raw, Raw) or raw.info.error is None or raw._past_layer_limit # pylint: disable=protected-access diff --git a/pcapkit/toolkit/pypcapfile.py b/pcapkit/toolkit/pypcapfile.py index b3766305f2..3c95b58a51 100644 --- a/pcapkit/toolkit/pypcapfile.py +++ b/pcapkit/toolkit/pypcapfile.py @@ -39,6 +39,12 @@ :class:`~pcapfile.protocols.transport.tcp.TCP` class rather than a reimplementation. +.. versionremoved:: 1.5.0 + + ``pcapkit.toolkit.pypcapfile.IPV6_TCP_WARNED_LIMIT``, now + :data:`pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT`, set with + :envvar:`PCAPKIT_IPV6_TCP_WARNED_LIMIT` (:issue:`1726`). + """ import binascii import collections @@ -55,9 +61,9 @@ from pcapkit.foundation.reassembly.data.ip import Packet as IP_Packet from pcapkit.foundation.reassembly.data.tcp import Packet as TCP_Packet from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet -from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT from pcapkit.protocols.internet.ipv4 import IPv4 as Protocol_IPv4 from pcapkit.protocols.transport.tcp import TCP as Protocol_TCP +from pcapkit.utilities import config from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall, stacklevel from pcapkit.utilities.warnings import AttributeWarning, warn @@ -96,12 +102,6 @@ #: Length of the fixed IPv6 header, :rfc:`8200#section-3`. IPV6_HEADER_LEN = 40 -#: How many captures :func:`_decline_ipv6_tcp` remembers having warned about. -#: Each one pins a packet of its own in memory (see :func:`_capture_of`), so the -#: memory is bounded and the oldest is forgotten first. Up to this many captures -#: read at the same time warn exactly once each. -IPV6_TCP_WARNED_LIMIT = 32 - #: The captures :func:`_decline_ipv6_tcp` has warned about, oldest first. Each #: key :func:`_capture_of` gives maps to the anchor that keeps that key unique. _ipv6_tcp_warned = collections.OrderedDict() # type: OrderedDict[int, Any] @@ -212,9 +212,10 @@ def _parse_mac_address(value: 'Any') -> 'str': field. Returns: - Lowercase, colon-separated hex MAC address, e.g. ``'01:00:5e:01:03:03'`` - -- the same form :meth:`~pcapkit.protocols.link.ethernet.Ethernet._read_mac_addr` - uses for the default engine. + The MAC address as text. A raw 6-octet ``value`` is formatted as + lowercase hex, two digits per octet, colon-separated, e.g. + ``'01:00:5e:01:03:03'``; any other is decoded as ASCII and returned + as is. Note: The released `PyPCAPFile`_ (0.12.0) already stores ``src``/``dst`` pre-formatted @@ -431,7 +432,7 @@ def _decline_ipv6_tcp(packet: 'Packet', count: 'int') -> 'None': capture to reach :func:`tcp_reassembly` or :func:`tcp_traceflow`. Later frames of the same capture are left out without another warning, and every other capture warns once of its own, however - their frames interleave. Only the :data:`IPV6_TCP_WARNED_LIMIT` + their frames interleave. Only the :data:`~pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT` captures warned about most recently are remembered, so with more than that read at once a forgotten one warns again: the warning can repeat, but is never omitted. @@ -444,7 +445,7 @@ def _decline_ipv6_tcp(packet: 'Packet', count: 'int') -> 'None': return _ipv6_tcp_warned[key] = anchor - while len(_ipv6_tcp_warned) > IPV6_TCP_WARNED_LIMIT: + while config.IPV6_TCP_WARNED_LIMIT != -1 and len(_ipv6_tcp_warned) > config.IPV6_TCP_WARNED_LIMIT: _ipv6_tcp_warned.popitem(last=False) warn(f"Frame {count}: TCP over IPv6 is left out of TCP reassembly and flow tracing, as " "'pypcapfile' has no IPv6 decoder; later such frames of this capture are left out " @@ -555,7 +556,7 @@ def _walk_extension_headers(protocol: 'int', data: 'bytes') -> 'Optional[tuple[i on, and how many headers were stepped over -- ``protocol``, ``data`` and ``0`` if ``protocol`` is none of them -- or :data:`None` if one of them is cut short or rejected, or more than - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` + :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` follow in a row. An AH is ``(Payload Length + 2) * 4`` octets long (:rfc:`4302#section-2.2`), and no shorter than :data:`AH_HEADER_LEN`; any other is read by :func:`_default_extension_header`. @@ -563,7 +564,7 @@ def _walk_extension_headers(protocol: 'int', data: 'bytes') -> 'Optional[tuple[i """ depth = 0 while protocol in IPV4_EXTENSION_HEADERS: - if depth == EXTENSION_HEADER_LIMIT: + if depth == config.EXTENSION_HEADER_LIMIT: return None depth += 1 if protocol == Enum_TransType.AH: diff --git a/pcapkit/toolkit/scapy.py b/pcapkit/toolkit/scapy.py index 19528b8c70..a70e9e13ee 100644 --- a/pcapkit/toolkit/scapy.py +++ b/pcapkit/toolkit/scapy.py @@ -389,7 +389,7 @@ def _tcp_carrier(packet: 'Packet') -> 'tuple[IP | IPv6, TCP, bytes, bytes] | Non An IPv4 layer whose protocol names an AH or IPv6 extension header (:data:`~pcapkit.toolkit.pypcapfile.IPV4_EXTENSION_HEADERS`) has those headers stepped over in its octets, up to - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of + :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` of them in a row (see :func:`~pcapkit.toolkit.pypcapfile._walk_extension_headers`), and what follows them is dissected from the octets: a tunnel, walked on as @@ -729,8 +729,7 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | No # put the moment of parsing into every flow label -- so the same # capture traced twice would produce different label strings and # different output filenames. Scapy carries the record's - # own timestamp on ``Packet.time``, which is what every other - # engine's adapter reports. + # own timestamp on ``Packet.time``, so that is what is reported. timestamp=float(packet.time), # capture timestamp seq=tcp.seq, # TCP sequence number ack=tcp.ack, # TCP acknowledgement number diff --git a/pcapkit/utilities/chardet.py b/pcapkit/utilities/chardet.py index a8cd7c67c3..32a14e65f1 100644 --- a/pcapkit/utilities/chardet.py +++ b/pcapkit/utilities/chardet.py @@ -9,20 +9,25 @@ .. _chardet: https://chardet.readthedocs.io +.. versionremoved:: 1.5.0 + + ``pcapkit.utilities.chardet.DETECT_CACHE_SIZE``, now + :data:`pcapkit.utilities.config.DETECT_CACHE_SIZE`, set with + :envvar:`PCAPKIT_DETECT_CACHE_SIZE` (:issue:`1726`). + """ import functools import chardet -__all__ = ['detect'] +from pcapkit.utilities import config -#: How many distinct bytestrings :func:`detect` will remember. Bounded so that a -#: capture full of never-repeating text cannot retain all of it. -DETECT_CACHE_SIZE = 1024 +__all__ = ['detect'] -@functools.lru_cache(maxsize=DETECT_CACHE_SIZE) +# NOTE: The size is read once, here, since it is fixed when the cache is built. +@functools.lru_cache(maxsize=None if config.DETECT_CACHE_SIZE == -1 else config.DETECT_CACHE_SIZE) def detect(value: 'bytes') -> 'str': """Detect the character set of ``value``. @@ -38,7 +43,7 @@ def detect(value: 'bytes') -> 'str': it was keyed on: :func:`functools.lru_cache` caches an argument's *hash* but still keeps the argument, since a :obj:`dict` needs the key to settle equality on a hash collision. Measured, feeding 20 distinct 1 MB values - retains 19.1 MB. :data:`DETECT_CACHE_SIZE` therefore caps the entries + retains 19.1 MB. :data:`~pcapkit.utilities.config.DETECT_CACHE_SIZE` therefore caps the entries rather than the footprint, which matters because :meth:`ProtocolBase.decode ` is public and a caller diff --git a/pcapkit/utilities/config.py b/pcapkit/utilities/config.py new file mode 100644 index 0000000000..eec6d2e63a --- /dev/null +++ b/pcapkit/utilities/config.py @@ -0,0 +1,419 @@ +# -*- coding: utf-8 -*- +"""Configuration +================= + +.. module:: pcapkit.utilities.config + +:mod:`pcapkit.utilities.config` holds every tunable limit of :mod:`pcapkit`, +and the settings of the :mod:`pcapkit.vendor` crawlers (:issue:`1726`). Each +limit is read from a ``PCAPKIT_*`` environment variable once, when this module +is imported, and validated there: + +* A value that is not an integer, or is below the variable's minimum, is + reported with a :exc:`~pcapkit.utilities.warnings.ConfigWarning`, and the + default is used instead. +* A limit takes ``-1`` to mean no limit at all. That, or any value above the + default, is accepted and reported with a + :exc:`~pcapkit.utilities.warnings.ConfigWarning` naming the risk. + +Each message names the variable and the value it was given. The warning is +reported through :func:`~pcapkit.utilities.warnings.warn`, so it obeys the +warning filters: under :option:`-W error <-W>` or ``PYTHONWARNINGS=error``, an +out-of-range value makes ``import pcapkit`` raise +:exc:`~pcapkit.utilities.warnings.ConfigWarning`. + +Every reader looks the value up here at call time, as ``config.``, so +patching one name in this module reaches every engine and every protocol that +reads it. The proxy addresses are the exception: :func:`http_proxy` and +:func:`https_proxy` read their variables on every call. + +Besides the standard library, this module imports +:mod:`pcapkit.utilities.warnings` and nothing else from :mod:`pcapkit`, which is +what lets :mod:`pcapkit.utilities.logging` load before it. The three flags that +:mod:`pcapkit.utilities.logging` needs while it is loading, +:envvar:`PCAPKIT_DEVMODE`, :envvar:`PCAPKIT_VERBOSE` and +:envvar:`PCAPKIT_SPHINX`, are therefore read there instead. + +.. versionadded:: 1.5.0 + +""" +import os +from typing import TYPE_CHECKING + +from pcapkit.utilities.warnings import ConfigWarning, _importer_stacklevel, warn + +if TYPE_CHECKING: + from typing import Optional + +__all__ = [ + # limits + 'FRAME_LAYER_LIMIT', 'EXTENSION_HEADER_LIMIT', 'IPV6_TCP_WARNED_LIMIT', + 'FALLBACK_LIMIT', 'DETECT_CACHE_SIZE', 'READ_CHUNK', + 'MAX_ZERO_PAD_LENGTH', 'MAX_ZERO_PAD_SHORTFALL', 'ZERO_PAD_BUDGET_RATIO', + # vendor crawlers + 'VENDOR_RETRY', 'CI_MODE', 'http_proxy', 'https_proxy', +] + +#: Strings a boolean variable may take, case-insensitively. +_BOOLEAN_STATES = {'1': True, '0': False, + 'yes': True, 'no': False, + 'true': True, 'false': False, + 'on': True, 'off': False} + +#: What a recursion-bound limit risks once lifted, c.f. :issue:`1610`. +_RECURSION_RISK = ("each layer is dissected from inside the parse of the one before it, so a deep " + "frame may hit Python's recursion limit and raise RecursionError (#1610)") + + +def _read_limit(name: 'str', default: 'int', *, minimum: 'int' = 0, + risk: 'Optional[str]' = None) -> 'int': + """Read the integer environment variable ``PCAPKIT_``. + + Args: + name: Variable name, without the ``PCAPKIT_`` prefix. + default: Value used when the variable is unset or invalid. + minimum: Smallest value accepted, besides ``-1`` where ``risk`` is set. + risk: What lifting the limit risks. If set, ``-1`` means no limit, and + ``-1`` or any value above ``default`` is reported with it. If + :data:`None`, ``-1`` is not accepted and nothing is reported. + + Returns: + The value of the variable, or ``default``. + + """ + variable = f'PCAPKIT_{name}' + text = os.environ.get(variable) + if text is None: + return default + + accepted = f'an integer of at least {minimum}' + (', or -1 for no limit' if risk is not None else '') + try: + value = int(text) + except ValueError: + value = None # type: ignore[assignment] + if value is None or (value < minimum and not (risk is not None and value == -1)): + warn(f'{variable}={text!r} is not {accepted}; using the default, {default}', + ConfigWarning, stacklevel=_importer_stacklevel()) + return default + + if risk is not None and (value == -1 or value > default): + lifted = 'lifts the limit' if value == -1 else f'is above the default of {default}' + warn(f'{variable}={text!r} {lifted}: {risk}', ConfigWarning, + stacklevel=_importer_stacklevel()) + return value + + +def _read_flag(name: 'str', default: 'bool' = False) -> 'bool': + """Read the boolean environment variable ``PCAPKIT_``. + + Args: + name: Variable name, without the ``PCAPKIT_`` prefix. + default: Value used when the variable is unset or invalid. + + Returns: + The value of the variable, or ``default``. + + """ + variable = f'PCAPKIT_{name}' + text = os.environ.get(variable) + if text is None: + return default + try: + return _BOOLEAN_STATES[text.strip().casefold()] + except KeyError: + warn(f'{variable}={text!r} is not one of {", ".join(_BOOLEAN_STATES)}; ' + f'using the default, {default}', ConfigWarning, stacklevel=_importer_stacklevel()) + return default + + +############################################################################### +# Limits +############################################################################### + +#: int: Most layers dissected in one frame (:issue:`1610`), +#: :envvar:`PCAPKIT_FRAME_LAYER_LIMIT`. +#: +#: A layer is one entry of the frame's protocol chain, counted from the first +#: layer the frame record carries, usually its link layer; a protocol +#: constructed from its own octets is not counted itself, so the count starts at +#: the first layer it dissects. IPv6 extension headers are layers like any +#: other, whether :class:`~pcapkit.protocols.internet.ipv6.IPv6` walks them or +#: each dissects the next. What follows the last layer is kept as +#: :class:`~pcapkit.protocols.misc.raw.Raw`, with one +#: :exc:`~pcapkit.utilities.warnings.ProtocolWarning`, unless it would have been +#: left raw anyway or there is nothing left. +#: +#: Each layer dissects the next from inside its own parse, so without the bound +#: a deep enough frame, such as IP in IP or stacked 802.1Q tags, stops wherever +#: Python's recursion limit falls, which depends on the interpreter and on how +#: deep the caller's own stack is. A tunnel does not restart the count, and nor +#: does the end of a chain of extension headers, which +#: :data:`EXTENSION_HEADER_LIMIT` bounds on its own. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`` (:issue:`1726`). +FRAME_LAYER_LIMIT = _read_limit('FRAME_LAYER_LIMIT', 64, risk=_RECURSION_RISK) + +#: int: Most IPv6 extension headers dissected in one chain, i.e. one after +#: another with no other layer between them (:issue:`1604`), +#: :envvar:`PCAPKIT_EXTENSION_HEADER_LIMIT`. +#: +#: The walk stops there, and what follows the last of them is kept as +#: :class:`~pcapkit.protocols.misc.raw.Raw`, with one +#: :exc:`~pcapkit.utilities.warnings.ProtocolWarning`. That holds whether the +#: chain follows IPv6, which walks it in a loop +#: (:meth:`IPv6._decode_next_layer +#: `), or another +#: layer such as IPv4, where each header dissects the next one itself +#: (:meth:`Internet._import_next_layer +#: `). Behind +#: IPv4, the ``scapy`` toolkit steps over at most this many in the octets to find +#: TCP; the third-party engines are otherwise best effort here (:issue:`1687`). +#: +#: :rfc:`8200#section-4.1` allows each extension header at most once, and the +#: Destination Options header at most twice, so no compliant chain gets close. +#: Without the bound, a chain dissected header by header stops wherever +#: Python's recursion limit falls, which depends on the interpreter. Each +#: header is a layer, so :data:`FRAME_LAYER_LIMIT` applies as well. +#: +#: Rebuilding a frame with :meth:`~pcapkit.protocols.protocol.ProtocolBase.from_data` +#: stops where parsing it did, and warns once too (:issue:`1619`), c.f. +#: :meth:`Internet._make_payload +#: `. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`` (:issue:`1726`). +EXTENSION_HEADER_LIMIT = _read_limit('EXTENSION_HEADER_LIMIT', 32, risk=_RECURSION_RISK) + +#: int: How many captures the ``pypcapfile`` engine remembers having warned +#: about for TCP over IPv6, :envvar:`PCAPKIT_IPV6_TCP_WARNED_LIMIT`. +#: +#: Each one pins a packet of its own in memory, so the memory is bounded and the +#: oldest is forgotten first. Up to this many captures read at the same time +#: warn exactly once each. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.toolkit.pypcapfile.IPV6_TCP_WARNED_LIMIT`` (:issue:`1726`). +IPV6_TCP_WARNED_LIMIT = _read_limit('IPV6_TCP_WARNED_LIMIT', 32, risk=( + 'each capture remembered pins one of its packets in memory')) + +#: int: Most objects :meth:`DictDumper._append_fallback +#: ` expands for one +#: top-level object, counting an object once per time it is met, +#: :envvar:`PCAPKIT_FALLBACK_LIMIT`; ``-1`` expands without bound. +#: +#: No fallback object in the ``foundation``, ``dumpkit`` or ``interface`` test +#: legs expands into more than 7, so the default leaves three orders of +#: magnitude of room. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.dumpkit.common.FALLBACK_LIMIT`` (:issue:`1726`). +FALLBACK_LIMIT = _read_limit('FALLBACK_LIMIT', 10_000, risk=( + 'an object that expands into a large or endless object graph is written out ' + 'in full, which takes memory and time without bound')) + +#: int: How many distinct bytestrings :func:`~pcapkit.utilities.chardet.detect` +#: remembers, :envvar:`PCAPKIT_DETECT_CACHE_SIZE`; ``-1`` leaves the cache +#: unbounded. +#: +#: Unlike the other limits, this one is read once, when +#: :mod:`pcapkit.utilities.chardet` is imported, as the size of its +#: :func:`functools.lru_cache`. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.utilities.chardet.DETECT_CACHE_SIZE`` (:issue:`1726`). +DETECT_CACHE_SIZE = _read_limit('DETECT_CACHE_SIZE', 1024, risk=( + 'the cache keeps every bytestring it was asked about, so a capture full of ' + 'never-repeating text is retained in memory')) + +#: int: Most octets :class:`~pcapkit.corekit.io.SeekableReader` asks of its +#: stream in one read, :envvar:`PCAPKIT_READ_CHUNK`; ``-1`` asks for the whole +#: read at once. +#: +#: A read allocates what it asks for up front, so a record's bogus length is read +#: in pieces. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.corekit.io._READ_CHUNK`` (:issue:`1726`). +READ_CHUNK = _read_limit('READ_CHUNK', 1 << 20, minimum=1, risk=( + 'a bogus record length allocates up to that many octets in one read')) + +#: int: Ceiling on the zero-padding :meth:`FieldBase.unpack +#: ` will still perform for a +#: field whose declared length outruns its buffer. +#: +#: This is libpcap's own ``MAXIMUM_SNAPLEN`` -- the point past which libpcap +#: itself treats a capture's declared snapshot length as corrupt or +#: byte-order-swapped rather than real -- and it is the same figure this +#: package's own PCAP writer defaults ``snaplen`` to +#: (:meth:`pcapkit.protocols.misc.pcap.header.Header.make`). No single field +#: within one captured packet is legitimately larger than the largest packet +#: libpcap itself is willing to believe, so nothing this library parses +#: should ever declare a length past it. +#: +#: The bound is deliberately *not* "any declared length beyond what the +#: buffer holds": :meth:`ListField.unpack ` and :meth:`OptionField.unpack ` depend on reading a short, sometimes +#: empty, tail past a truncated area and having it decode as zero -- that is +#: how an over-long ``ihl``, or a capture cut short by the snapshot length, +#: reads as end-of-option-list or ``Pad1`` instead of wedging or raising (see +#: :issue:`431`). Every such read is of a fixed-width, few-octet field, always +#: far under this ceiling, so it is untouched; only a length past it -- which +#: no fixed-width field ever legitimately is -- gets refused. +#: +#: A ceiling on one field says nothing about how many fields a parse may pad, +#: which is what :data:`MAX_ZERO_PAD_SHORTFALL` and +#: :data:`ZERO_PAD_BUDGET_RATIO` below are for. See :issue:`573`. +#: +#: Set with :envvar:`PCAPKIT_MAX_ZERO_PAD_LENGTH`. ``-1`` lifts the ceiling, and +#: lifts the running budget of :data:`ZERO_PAD_BUDGET_RATIO` with it, since the +#: ceiling is the one-off term of that budget's allowance. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.corekit.fields.field._MAX_ZERO_PAD_LENGTH`` (:issue:`1726`). +MAX_ZERO_PAD_LENGTH = _read_limit('MAX_ZERO_PAD_LENGTH', 0x40_000, risk=( + 'a hostile capture can declare a field that long and have it zero-filled in memory')) + +#: int: Shortfall :meth:`FieldBase.unpack +#: ` will always zero-pad for, +#: whatever else a parse has already padded. +#: +#: :data:`MAX_ZERO_PAD_LENGTH` bounds each field on its own, and a packet holds +#: many fields, so it leaves the *sum* unbounded: a declared length just under the +#: ceiling is honoured however often it is declared. Measured on this tree, 200 +#: minimal PCAP-NG Decryption Secrets Blocks -- 4,800 wire octets, each block +#: declaring ``secrets_length`` of 262,142 against two supplied octets through +#: ``UnknownSecrets.data`` (``pcapkit/protocols/schema/misc/pcapng.py``) -- +#: retained 50.0 MiB, an amplification of 10,922x per block. Every individual +#: field was under the ceiling, so nothing refused any of them (:issue:`573`). +#: +#: The sum therefore wants a budget, and this is the figure that makes one +#: *safe*. A budget on its own is not: a capture cut short by its snapshot length +#: pads legitimately and must keep parsing (:issue:`431`, and the reasoning that +#: declined :issue:`554`), and it pads far more than it reads, so any running +#: budget tight enough to matter starts refusing real captures. Worse, it refuses +#: them *sometimes* -- measured on this tree with a running budget alone, the +#: same legitimate 54-octet frame parsed to one result on 37 of 40 calls and to +#: another on calls 26, 33 and 39, because whether it fit depended on what had +#: been parsed before it. A guard whose answer moves with history is not a guard. +#: +#: 65,536 is what removes that. It is the whole span of a 16-bit wire length +#: field -- which is how an IP header, an IPv6 payload, a TCP or IPv4 option and +#: a PCAP-NG option all declare their size -- so no shortfall that one of those +#: can produce is subject to the budget at all, and every one of them is padded +#: unconditionally. Measured: the largest legitimate single +#: shortfall found anywhere was 65,495 octets, from a snapshot-truncated +#: offload-sized frame (100 frames at ``incl_len`` 54 declaring an IPv4 total +#: length of 65,535, padding 6,549,500 octets from 7,024 read), and 64,750 from a +#: truncated PCAP-NG option. Both sit under this figure, and so does anything +#: else a 16-bit field can ask for. +#: +#: What is left above it is the band a *32-bit* wire length reaches -- +#: PCAP-NG's own block and secrets lengths, which is where :issue:`573`'s +#: amplification lives -- and legitimately that is a once-per-file event, since +#: only the last block of a truncated capture is cut short. So the band gets the +#: running budget below, whose one-off term already covers any single such event +#: outright. +#: +#: Set with :envvar:`PCAPKIT_MAX_ZERO_PAD_SHORTFALL`. ``-1`` disables the running +#: budget: every shortfall within :data:`MAX_ZERO_PAD_LENGTH` is then padded +#: without being charged to it. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.corekit.fields.field._MAX_ZERO_PAD_SHORTFALL`` (:issue:`1726`). +MAX_ZERO_PAD_SHORTFALL = _read_limit('MAX_ZERO_PAD_SHORTFALL', 0x10_000, risk=( + 'shortfalls up to that size are zero-filled without being charged to the running budget, ' + 'so a capture of many lying lengths can amplify into that much memory each')) + +#: int: Zero padding *past* :data:`MAX_ZERO_PAD_SHORTFALL` that +#: :meth:`FieldBase.unpack ` +#: will synthesise in total for every octet a parse has actually been given, over +#: and above the one-off :data:`MAX_ZERO_PAD_LENGTH` allowance. +#: +#: A shortfall in this band -- past 65,536 octets and so past anything a 16-bit +#: wire length can declare, but within :data:`MAX_ZERO_PAD_LENGTH` -- is a +#: PCAP-NG block or secrets length, a 32-bit figure. One of those, on its own, is +#: legitimate: it is what the final block of a capture truncated at EOF looks +#: like, and the ``MAX_ZERO_PAD_LENGTH`` term of the allowance covers it whole, +#: from a cold start, because such a shortfall cannot exceed that ceiling and +#: still be padded at all. +#: +#: *Repeating* one is not legitimate, and that is what 16 is chosen to catch. +#: Truncation cuts the end of a file, so a capture has one short block, not two +#: hundred; :issue:`573`'s shape has two hundred because they are declared +#: rather than cut. 16 octets of further allowance per octet genuinely read +#: leaves any real file an allowance orders of magnitude past the one event it +#: can want, while bounding the sum for a file whose blocks all lie. +#: +#: Set with :envvar:`PCAPKIT_ZERO_PAD_BUDGET_RATIO`. ``-1`` lifts the running +#: budget; :data:`MAX_ZERO_PAD_LENGTH` still bounds each field. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.corekit.fields.field._ZERO_PAD_BUDGET_RATIO`` (:issue:`1726`). +ZERO_PAD_BUDGET_RATIO = _read_limit('ZERO_PAD_BUDGET_RATIO', 0x10, risk=( + 'a capture whose blocks all lie about their lengths can amplify each octet read ' + 'into that many zero octets in memory')) + +############################################################################### +# Vendor Crawlers +############################################################################### + +#: int: How many times a :mod:`pcapkit.vendor` crawler retries a failed request, +#: :envvar:`PCAPKIT_VENDOR_RETRY`. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.vendor.default.MAX_RETRY`` (:issue:`1726`), which +#: crashed the import on a value that was not an integer, and read ``0`` as +#: ``1``. +VENDOR_RETRY = _read_limit('VENDOR_RETRY', 5, minimum=1) + +#: bool: Whether a :mod:`pcapkit.vendor` crawler runs in CI, where a failed +#: fetch fails the run instead of prompting, :envvar:`PCAPKIT_CI_MODE`. +#: +#: .. versionadded:: 1.5.0 +#: +#: Moved from ``pcapkit.vendor.default.CI_MODE`` (:issue:`1726`). +CI_MODE = _read_flag('CI_MODE') + + +def http_proxy() -> 'Optional[str]': + """HTTP proxy address for the :mod:`pcapkit.vendor` crawlers. + + Unlike the limits, :envvar:`PCAPKIT_HTTP_PROXY` is read on every call, as + :func:`~pcapkit.vendor.default.get_proxies` always has, so a proxy set after + :mod:`pcapkit` is imported is still honoured. + + Returns: + The value of :envvar:`PCAPKIT_HTTP_PROXY`, or :data:`None` if unset. + + .. versionadded:: 1.5.0 + + """ + return os.environ.get('PCAPKIT_HTTP_PROXY') + + +def https_proxy() -> 'Optional[str]': + """HTTPS proxy address for the :mod:`pcapkit.vendor` crawlers. + + Unlike the limits, :envvar:`PCAPKIT_HTTPS_PROXY` is read on every call, as + :func:`~pcapkit.vendor.default.get_proxies` always has, so a proxy set after + :mod:`pcapkit` is imported is still honoured. + + Returns: + The value of :envvar:`PCAPKIT_HTTPS_PROXY`, or :data:`None` if unset. + + .. versionadded:: 1.5.0 + + """ + return os.environ.get('PCAPKIT_HTTPS_PROXY') diff --git a/pcapkit/utilities/logging.py b/pcapkit/utilities/logging.py index 517668d200..ed18735e52 100644 --- a/pcapkit/utilities/logging.py +++ b/pcapkit/utilities/logging.py @@ -60,17 +60,47 @@ 'true': True, 'false': False, 'on': True, 'off': False} +#: list[tuple[str, str]]: Each flag below whose value is not one of +#: :data:`BOOLEAN_STATES`, as ``(variable, value)``, in the order read. +#: +#: This module loads before :mod:`pcapkit.utilities.warnings`, which needs +#: :data:`VERBOSE`, so it cannot warn of a bad value itself. It records the +#: value here instead, and :mod:`pcapkit.utilities.warnings` reports each with a +#: :exc:`~pcapkit.utilities.warnings.ConfigWarning` once it has loaded +#: (:issue:`1726`). +_BAD_FLAGS = [] # type: list[tuple[str, str]] + + +def _read_flag(variable: 'str') -> 'bool': + """Read the boolean environment variable ``variable``, off by default. + + The value is read with surrounding whitespace stripped and + case-insensitively, as :func:`pcapkit.utilities.config._read_flag` reads + one. A value that is not then one of :data:`BOOLEAN_STATES` is taken as off + and recorded in :data:`_BAD_FLAGS`. + + """ + text = os.environ.get(variable) + if text is None: + return False + try: + return BOOLEAN_STATES[text.strip().casefold()] + except KeyError: + _BAD_FLAGS.append((variable, text)) + return False + + #: bool: Development mode flag. -DEVMODE = BOOLEAN_STATES.get(os.environ.get('PCAPKIT_DEVMODE', 'false').casefold(), False) +DEVMODE = _read_flag('PCAPKIT_DEVMODE') #: bool: Verbose output flag. -VERBOSE = BOOLEAN_STATES.get(os.environ.get('PCAPKIT_VERBOSE', 'false').casefold(), False) +VERBOSE = _read_flag('PCAPKIT_VERBOSE') ############################################################################### # Sphinx Mode ############################################################################### #: bool: This is a workaround for :data:`typing.TYPE_CHECKING` in Sphinx. -SPHINX_TYPE_CHECKING = BOOLEAN_STATES.get(os.environ.get('PCAPKIT_SPHINX', 'false').casefold(), False) +SPHINX_TYPE_CHECKING = _read_flag('PCAPKIT_SPHINX') ############################################################################### # Logger Setup diff --git a/pcapkit/utilities/warnings.py b/pcapkit/utilities/warnings.py index 62fd211ab1..f6665ad064 100644 --- a/pcapkit/utilities/warnings.py +++ b/pcapkit/utilities/warnings.py @@ -58,11 +58,12 @@ """ import contextvars import inspect +import os import warnings from typing import TYPE_CHECKING from pcapkit.utilities.exceptions import stacklevel as stacklevel_calculator -from pcapkit.utilities.logging import VERBOSE, get_logger +from pcapkit.utilities.logging import _BAD_FLAGS, BOOLEAN_STATES, VERBOSE, get_logger if TYPE_CHECKING: from types import FrameType, TracebackType @@ -85,7 +86,7 @@ 'FileWarning', 'LayerWarning', 'ProtocolWarning', 'AttributeWarning', 'DevModeWarning', 'VendorRequestWarning', 'VendorRuntimeWarning', 'UnknownFieldWarning', 'RegistryWarning', 'SchemaWarning', 'InfoWarning', - 'SeekWarning', 'ExtractionWarning', + 'SeekWarning', 'ExtractionWarning', 'ConfigWarning', # ResourceWarning 'DPKTWarning', 'ScapyWarning', 'PySharkWarning', 'EmojiWarning', 'VendorWarning', @@ -407,6 +408,18 @@ class ExtractionWarning(BaseWarning, RuntimeWarning): """Extraction warning.""" +class ConfigWarning(BaseWarning, RuntimeWarning): + """Configuration warning. + + Reported by :mod:`pcapkit.utilities.config` for an environment variable that + is invalid, and so falls back to its default, or that lifts a limit past its + default. + + .. versionadded:: 1.5.0 + + """ + + ############################################################################## # ResourceWarning session. ############################################################################## @@ -439,3 +452,59 @@ class VendorWarning(BaseWarning, ResourceWarning): class DeprecatedFormatWarning(BaseWarning, DeprecationWarning): """Warning on deprecated formats.""" + + +############################################################################### +# Deferred Reports +############################################################################### + + +def _importer_stacklevel() -> 'int': + """Stack level, relative to the caller, of the innermost frame outside :mod:`pcapkit`. + + For a :exc:`ConfigWarning` given while :mod:`pcapkit` is imported, which + then names the module that imported it (:issue:`1726`). + :func:`~pcapkit.utilities.exceptions.stacklevel` counts every frame, but + :func:`warnings.warn` does not count those of :mod:`importlib`'s bootstrap, + which an import puts between the importing module and :mod:`pcapkit`. Its + level then runs past the outermost frame, and the warning is attributed to + ``:0``. This counts frames as :func:`warnings.warn` does, and + :mod:`logging` does from Python 3.11; on earlier versions the log record is + attributed to a frame inside the import. + + """ + pcapkit = f'{os.path.sep}pcapkit{os.path.sep}' + boundary = outermost = level = 0 + frame = inspect.currentframe() + try: + frame = frame.f_back if frame is not None else None # the caller, level 1 + while frame is not None: + filename = frame.f_code.co_filename + if not ('importlib' in filename and '_bootstrap' in filename): + level += 1 + outermost = level + if pcapkit in filename: + boundary = level + frame = frame.f_back + finally: + del frame + return max(1, min(boundary + 1, outermost)) + + +def _report_bad_flags() -> 'None': + """Report the bad flags :mod:`pcapkit.utilities.logging` recorded. + + Each entry of :data:`pcapkit.utilities.logging._BAD_FLAGS` is reported with + a :exc:`ConfigWarning` and then dropped, so it is reported once. An entry is + dropped only after its warning returns, so under :option:`-W error <-W>` the + next import raises again rather than passing silently. + + """ + while _BAD_FLAGS: + variable, text = _BAD_FLAGS[0] + warn(f'{variable}={text!r} is not one of {", ".join(BOOLEAN_STATES)}; ' + 'using the default, False', ConfigWarning, stacklevel=_importer_stacklevel()) + del _BAD_FLAGS[0] + + +_report_bad_flags() diff --git a/pcapkit/vendor/default.py b/pcapkit/vendor/default.py index 6134617be3..838e2f33d5 100644 --- a/pcapkit/vendor/default.py +++ b/pcapkit/vendor/default.py @@ -7,6 +7,17 @@ :mod:`pcapkit.vendor.default` contains :class:`~pcapkit.vendor.default.Vendor` only, which is the base meta class for all vendor crawlers. +.. versionremoved:: 1.5.0 + + Moved to :mod:`pcapkit.utilities.config` (:issue:`1726`): + + * ``pcapkit.vendor.default.MAX_RETRY``, now + :data:`pcapkit.utilities.config.VENDOR_RETRY`, set with + :envvar:`PCAPKIT_VENDOR_RETRY`; + * ``pcapkit.vendor.default.CI_MODE``, now + :data:`pcapkit.utilities.config.CI_MODE`, set with + :envvar:`PCAPKIT_CI_MODE`. + """ import abc import collections @@ -26,8 +37,8 @@ import requests from pcapkit import __version__ +from pcapkit.utilities import config from pcapkit.utilities.exceptions import VendorNotImplemented, VendorPathNotFound -from pcapkit.utilities.logging import BOOLEAN_STATES from pcapkit.utilities.warnings import VendorRequestWarning, warn if TYPE_CHECKING: @@ -36,9 +47,6 @@ __all__ = ['Vendor'] -MAX_RETRY = int(os.environ.get('PCAPKIT_VENDOR_RETRY', 5)) or 1 -CI_MODE = BOOLEAN_STATES.get(os.environ.get('PCAPKIT_CI_MODE', 'false').casefold(), False) - #: Distribution name of this package, i.e. the name its metadata is registered #: under, which is not the import name (:mod:`pcapkit`). DISTRIBUTION = 'pypcapkit' @@ -93,9 +101,11 @@ def _missing_(cls, value: 'int') -> '{NAME}': def get_proxies() -> 'dict[str, str]': """Get proxy for blocked sites. - The function will read :envvar:`PCAPKIT_HTTP_PROXY` - and :envvar:`PCAPKIT_HTTPS_PROXY`, if any, for the - proxy settings of |requests|_. + The function will read :envvar:`PCAPKIT_HTTP_PROXY` and + :envvar:`PCAPKIT_HTTPS_PROXY`, if any, on every call, through + :func:`~pcapkit.utilities.config.http_proxy` and + :func:`~pcapkit.utilities.config.https_proxy`, for the proxy settings of + |requests|_. .. |requests| replace:: ``requests`` .. _requests: https://requests.readthedocs.io @@ -104,8 +114,8 @@ def get_proxies() -> 'dict[str, str]': Proxy settings for |requests|_. """ - HTTP_PROXY = os.getenv('PCAPKIT_HTTP_PROXY') - HTTPS_PROXY = os.getenv('PCAPKIT_HTTPS_PROXY') + HTTP_PROXY = config.http_proxy() + HTTPS_PROXY = config.https_proxy() PROXIES = {} # type: dict[str, str] if HTTP_PROXY is not None: PROXIES['http'] = HTTP_PROXY @@ -597,19 +607,19 @@ def _request(self) -> 'list[str]': # NOTE: both branches below send this. Wikimedia rejects ``requests``' # default agent with HTTP 403, so a crawler without it fetches 126 bytes - # of robot-policy text and retries MAX_RETRY times against a refusal + # of robot-policy text and retries config.VENDOR_RETRY times against a refusal # that no amount of retrying will lift; see #518. headers = {'User-Agent': get_user_agent()} try: counter = 1 while True: - if counter > MAX_RETRY: + if counter > config.VENDOR_RETRY: raise requests.exceptions.RequestException page = requests.get(self.LINK, headers=headers) # nosec: B113 if not page.ok or not page.text: - warn(f'Connection failed; retry for {counter}/{MAX_RETRY}...', + warn(f'Connection failed; retry for {counter}/{config.VENDOR_RETRY}...', VendorRequestWarning, stacklevel=2) counter += 1 @@ -626,19 +636,19 @@ def _request(self) -> 'list[str]': counter = 1 while True: - if counter > MAX_RETRY: + if counter > config.VENDOR_RETRY: raise page = requests.get(self.LINK, headers=headers, proxies=proxies) # nosec: B113 if not page.ok or not page.text: - warn(f'Connection failed; retry with proxy for {counter}/{MAX_RETRY}...', + warn(f'Connection failed; retry with proxy for {counter}/{config.VENDOR_RETRY}...', VendorRequestWarning, stacklevel=2) counter += 1 continue break except requests.RequestException as error: - if CI_MODE: + if config.CI_MODE: warn('Connection failed; exit on CI mode...', VendorRequestWarning, stacklevel=2) raise @@ -647,7 +657,7 @@ def _request(self) -> 'list[str]': # NOTE: manual intervention needs an operator, and a process # with no terminal has none -- nobody to read the instructions # below, nobody to save the page, so the wait it ends in could - # only ever time out the whole run. Fail exactly as CI_MODE + # only ever time out the whole run. Fail exactly as config.CI_MODE # does rather than prompting into the void; see #522. warn('Connection failed; exit as stdin is not interactive...', VendorRequestWarning, stacklevel=2) diff --git a/pcapkit/vendor/pcapng/block_type.py b/pcapkit/vendor/pcapng/block_type.py index bb9009ab97..9bc1bd5f34 100644 --- a/pcapkit/vendor/pcapng/block_type.py +++ b/pcapkit/vendor/pcapng/block_type.py @@ -34,7 +34,7 @@ # The dead ``-02`` URL, # https://www.ietf.org/staging/draft-tuexen-opsawg-pcapng-02.html, serves a # 77968-byte HTML error page rather than a clean refusal, so ``Vendor._request`` -# rejects it on ``page.ok`` and retries MAX_RETRY times against a page that will +# rejects it on ``page.ok`` and retries config.VENDOR_RETRY times against a page that will # never come back. On ``-02`` the registries are ASCII art inside ``
`` and
 # the only HTML ```` is the running-header one, so
 # ``soup.select('table#table-9')`` below finds nothing and raises ``IndexError``.
diff --git a/pyproject.toml b/pyproject.toml
index dfd6387873..aeebb82214 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -277,7 +277,7 @@ dev = [
     "requests[socks]", "beautifulsoup4[html5lib]",
 ]
 docs = [
-    "Sphinx>=6.1.3", "furo",
+    "Sphinx>=7.3", "furo",
     "sphinx-autodoc-typehints", "sphinxext-opengraph", "sphinx-copybutton",
     "sphinxcontrib-mermaid",
     "typing-extensions", "mypy-extensions",
diff --git a/tests/corekit/test_fields_field.py b/tests/corekit/test_fields_field.py
index 232aa2a808..d79722f0a0 100644
--- a/tests/corekit/test_fields_field.py
+++ b/tests/corekit/test_fields_field.py
@@ -28,7 +28,7 @@ class FieldBaseUnpackBoundsTests(unittest.TestCase):
     ``Pad1`` instead of wedging or raising (#431). Rejecting every shortfall
     regardless of size would turn every one of those into a hard failure.
 
-    So the guard only fires past :data:`~pcapkit.corekit.fields.field._MAX_ZERO_PAD_LENGTH`
+    So the guard only fires past :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`
     -- libpcap's own ``MAXIMUM_SNAPLEN`` and this package's own default
     ``snaplen`` -- which every fixed-width field ``ListField``/``OptionField``
     read past EOF is nowhere near, and which no legitimate single field is
@@ -50,7 +50,9 @@ def setUp(self) -> None:
 
         self.BytesField = BytesField
         self.FieldValueError = FieldValueError
-        self.ceiling = field_module._MAX_ZERO_PAD_LENGTH
+        from pcapkit.utilities import config
+
+        self.ceiling = config.MAX_ZERO_PAD_LENGTH
 
     def test_exact_length_buffer_unpacks_unchanged(self) -> None:
         field = self.BytesField(length=4)
@@ -70,7 +72,7 @@ def test_small_field_short_by_one_octet_still_zero_pads(self) -> None:
         were never read -- rather than raise, or every capture cut short
         by its snapshot length would start failing to parse instead of
         reporting the tail as padding. This field is nowhere near
-        :data:`~pcapkit.corekit.fields.field._MAX_ZERO_PAD_LENGTH`, so the
+        :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`, so the
         shortfall must still be padded exactly as before #554.
 
         The zeros land on the *tail*, which is where the unread octets were:
@@ -189,7 +191,7 @@ class FieldBaseCumulativePaddingBudgetTests(unittest.TestCase):
 
     #554's ceiling is per-field, and #573 is what that leaves behind: a packet
     holds many fields, so a declared length sitting just *under*
-    :data:`~pcapkit.corekit.fields.field._MAX_ZERO_PAD_LENGTH` is honoured
+    :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH` is honoured
     however many times it is declared, and the sum has no bound at all. Measured
     on the tree that carried only the per-field ceiling, 200 minimal PCAP-NG
     Decryption Secrets Blocks -- 4,800 wire octets, each declaring
@@ -200,8 +202,8 @@ class FieldBaseCumulativePaddingBudgetTests(unittest.TestCase):
     :meth:`FieldBase.unpack ` now
     keeps a running ledger of octets supplied against octets synthesised, and
     refuses a shortfall *past*
-    :data:`~pcapkit.corekit.fields.field._MAX_ZERO_PAD_SHORTFALL` once the total
-    of those passes ``_MAX_ZERO_PAD_LENGTH + _ZERO_PAD_BUDGET_RATIO * supplied``.
+    :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL` once the total
+    of those passes ``MAX_ZERO_PAD_LENGTH + ZERO_PAD_BUDGET_RATIO * supplied``.
 
     Two things these tests have to hold at once, and the second is the harder.
 
@@ -215,7 +217,7 @@ class FieldBaseCumulativePaddingBudgetTests(unittest.TestCase):
     on 37 of 40 identical calls and to a different one on calls 26, 33 and 39,
     purely because of what had been parsed before it.
 
-    :data:`~pcapkit.corekit.fields.field._MAX_ZERO_PAD_SHORTFALL` is what removes
+    :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_SHORTFALL` is what removes
     that: a shortfall within the span of a 16-bit wire length is padded
     unconditionally and charged to nothing, so no shortfall a snapshot-truncated
     capture, a truncated option area or an over-long ``ihl`` can produce is ever
@@ -234,7 +236,10 @@ def setUp(self) -> None:
         self.field_module = field_module
         self.BytesField = BytesField
         self.FieldValueError = FieldValueError
-        self.ceiling = field_module._MAX_ZERO_PAD_LENGTH
+        from pcapkit.utilities import config
+
+        self.config = config
+        self.ceiling = config.MAX_ZERO_PAD_LENGTH
 
         # The ledger is cumulative per context, and this class's tests share one
         # import of the module, so an earlier test has already earned allowance
@@ -281,7 +286,7 @@ def test_many_fields_each_under_the_ceiling_are_refused_in_total(self) -> None:
         """The #573 scenario: 200 fields, every one of them within the ceiling.
 
         Each declares two octets short of
-        :data:`~pcapkit.corekit.fields.field._MAX_ZERO_PAD_LENGTH` against two
+        :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH` against two
         supplied octets, so #554's per-field guard is *correct* to let each one
         through -- and before #573 all 200 went through, retaining 50.0 MiB from
         4,800 octets of input. At least one has to be refused now, or the sum is
@@ -432,11 +437,11 @@ def test_the_unconditional_shortfall_is_65536_and_the_ratio_is_16(self) -> None:
         covers whole. A test that only read the constants back from the module
         would still pass if either figure were changed to something unjustified.
         """
-        self.assertEqual(self.field_module._MAX_ZERO_PAD_SHORTFALL, 65536)
+        self.assertEqual(self.config.MAX_ZERO_PAD_SHORTFALL, 65536)
         self.assertEqual(2 ** 16, 65536)
         self.assertGreater(65536, 65495)  # the worst legitimate shortfall measured
 
-        self.assertEqual(self.field_module._ZERO_PAD_BUDGET_RATIO, 16)
+        self.assertEqual(self.config.ZERO_PAD_BUDGET_RATIO, 16)
 
         # and the bands have to be the right way round, or the 32-bit band would
         # swallow the 16-bit one.
@@ -549,6 +554,38 @@ def test_a_field_that_is_not_short_is_never_refused_by_the_budget(self) -> None:
             self.assertEqual(ledger[1], 0)
             self.assertEqual(ledger[0], 500 * size)
 
+    def all_padded(self, *reads: 'tuple[int, int]') -> bool:
+        """Whether every ``(declared, supplied)`` read is padded, in one fresh budget."""
+        with self.field_module._zero_pad_budget():
+            for declared, supplied in reads:
+                try:
+                    self.BytesField(length=declared).unpack(bytes(supplied), {})
+                except self.FieldValueError:
+                    return False
+        return True
+
+    def test_the_budget_is_refused_one_octet_past_the_allowance(self) -> None:
+        """Padding up to the allowance is taken, and one octet past it is refused.
+
+        Two reads of 10 octets each earn ``262,144 + 16 * 20 = 262,464`` octets.
+        The first pads 70,000 of them, so the second may pad 192,464 and no more.
+        """
+        first = (70_010, 10)
+        for padding, padded in ((192_463, True), (192_464, True), (192_465, False)):
+            with self.subTest(padding=padding):
+                self.assertIs(self.all_padded(first, (padding + 10, 10)), padded)
+
+    def test_a_shortfall_past_65536_is_charged_and_65536_is_not(self) -> None:
+        """The band edge: a 65,536-octet shortfall is free, and 65,537 is charged.
+
+        The first read pads 262,143 octets from none read, one short of the
+        allowance, so a second read that is charged at all is refused.
+        """
+        first = (262_143, 0)
+        for padding, padded in ((65_535, True), (65_536, True), (65_537, False)):
+            with self.subTest(padding=padding):
+                self.assertIs(self.all_padded(first, (padding, 0)), padded)
+
 
 class FieldBaseShortReadPaddingSideTests(unittest.TestCase):
     """Which *side* the short-read zero padding goes on, for both byte orders.
diff --git a/tests/dumpkit/test_fallback_cycle_unit.py b/tests/dumpkit/test_fallback_cycle_unit.py
index a1db1834cd..13d4cea895 100644
--- a/tests/dumpkit/test_fallback_cycle_unit.py
+++ b/tests/dumpkit/test_fallback_cycle_unit.py
@@ -107,7 +107,9 @@ def chain(depth: 'int') -> 'Node':
         with self.assertRaisesRegex(UnsupportedCall, 'more than 10000 objects'):
             self.dump('JSON', {'node': chain(14)})
 
-        with mock.patch.object(common, 'FALLBACK_LIMIT', 7):
+        from pcapkit.utilities import config
+
+        with mock.patch.object(config, 'FALLBACK_LIMIT', 7):
             # seven objects: within the limit
             record = json.loads(self.dump('JSON', {'node': chain(2)}))['x']
             self.assertEqual(record['node']['right']['left'], {'leaf': 0})
diff --git a/tests/project/test_no_engine_agreement_prose_1708_unit.py b/tests/project/test_no_engine_agreement_prose_1708_unit.py
index 766df855ba..e1e24f3aa6 100644
--- a/tests/project/test_no_engine_agreement_prose_1708_unit.py
+++ b/tests/project/test_no_engine_agreement_prose_1708_unit.py
@@ -93,18 +93,6 @@
     ('pcapkit/protocols/schema/internet/ipv6.py', 'as the default engine walks'),
 }
 
-#: ``(path, phrase)`` of each agreement claim in a file the open change for
-#: issue #1726 owns, with the reason it is left to that change. Not checked
-#: for staleness: the entry excuses that phrase alone, and goes once the claim
-#: does.
-PENDING = {
-    ('pcapkit/protocols/internet/internet.py', 'stop at the same depth'):
-        'protocols/internet/internet.py is owned by the change for #1726',
-    ('pcapkit/toolkit/scapy.py', 'every other engine'):
-        'toolkit/scapy.py is owned by the change for #1726',
-}
-
-
 #: Whitespace, with the ``#`` or ``#:`` that opens a continued comment line,
 #: read as one space; and reST markup -- a role prefix such as ``:obj:``, a
 #: backtick run with the ``~`` it may carry, a hyperlink's trailing ``_`` --
@@ -175,7 +163,7 @@ def test_no_prose_claims_an_engine_agrees_with_another(self) -> None:
         for path in scanned():
             relative = path.relative_to(ROOT).as_posix()
             for lineno, name, phrase in claims(path.read_text(encoding='utf-8')):
-                if (relative, phrase) not in EXEMPT.union(PENDING):
+                if (relative, phrase) not in EXEMPT:
                     found.append(f'{relative}:{lineno}: [{name}] {phrase!r}')
         self.assertEqual(
             found, [],
diff --git a/tests/protocols/internet/test_ext_chain_limit_rebuild_1619_unit.py b/tests/protocols/internet/test_ext_chain_limit_rebuild_1619_unit.py
index 2f9a838381..c8141a1dba 100644
--- a/tests/protocols/internet/test_ext_chain_limit_rebuild_1619_unit.py
+++ b/tests/protocols/internet/test_ext_chain_limit_rebuild_1619_unit.py
@@ -2,7 +2,7 @@
 """Rebuilding a frame cut at the extension header chain limit warns once.
 
 GitHub issue #1619: #1604 bounded a chain of IPv6 extension headers at
-:data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` headers,
+:data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` headers,
 with one :exc:`~pcapkit.utilities.warnings.ProtocolWarning`, and parsing a cut
 frame does warn once. But each layer :meth:`from_data
 ` rebuilds parses its own
diff --git a/tests/protocols/internet/test_ipv6_ext_chain_limit_1604_unit.py b/tests/protocols/internet/test_ipv6_ext_chain_limit_1604_unit.py
index d57dd8b28b..0b900dfd00 100644
--- a/tests/protocols/internet/test_ipv6_ext_chain_limit_1604_unit.py
+++ b/tests/protocols/internet/test_ipv6_ext_chain_limit_1604_unit.py
@@ -9,7 +9,7 @@
 and 90 on 3.14, from a script at the default limit -- and IPv6, which walks its
 chain in a loop, went on to any depth.
 
-Now both stop at :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`
+Now both stop at :data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT`
 headers in a row: what follows is kept as
 :class:`~pcapkit.protocols.misc.raw.Raw`, with one
 :exc:`~pcapkit.utilities.warnings.ProtocolWarning`, however long the chain and
@@ -123,9 +123,9 @@ def summary(self, data: bytes) -> 'tuple[object, ...]':
         return (str(parsed.protochain), layer.info.protocol, len(layer.data), tuple(messages))
 
     def test_the_limit_is_32(self) -> None:
-        from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT
+        from pcapkit.utilities import config
 
-        self.assertEqual(EXTENSION_HEADER_LIMIT, 32)
+        self.assertEqual(config.EXTENSION_HEADER_LIMIT, 32)
 
     def test_a_chain_of_32_is_dissected_to_the_tcp(self) -> None:
         for carrier in ETHERTYPE:
diff --git a/tests/protocols/test_frame_layer_limit_1610_unit.py b/tests/protocols/test_frame_layer_limit_1610_unit.py
index af05a6832a..a1fea376a8 100644
--- a/tests/protocols/test_frame_layer_limit_1610_unit.py
+++ b/tests/protocols/test_frame_layer_limit_1610_unit.py
@@ -9,7 +9,7 @@
 chain of extension headers, but its count restarts at every tunnel.
 
 Now every frame stops at
-:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers, counted along its
+:data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT` layers, counted along its
 protocol chain from the first layer its record carries, through tunnels, tags
 and extension headers alike: what follows is kept as
 :class:`~pcapkit.protocols.misc.raw.Raw`, with one
@@ -182,9 +182,9 @@ def assert_cut(self, layers: 'list[str]', parsed: object, messages: 'list[str]')
         self.assertEqual(int(layer.info.protocol), named[layers[63]])  # type: ignore[attr-defined]
 
     def test_the_limit_is_64(self) -> None:
-        from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT
+        from pcapkit.utilities import config
 
-        self.assertEqual(FRAME_LAYER_LIMIT, 64)
+        self.assertEqual(config.FRAME_LAYER_LIMIT, 64)
 
     def test_a_frame_of_64_layers_is_dissected_to_the_tcp(self) -> None:
         # Guard: below the old recursion cut-off, so this held before #1610 too.
diff --git a/tests/toolkit/test_pypcapfile_unit.py b/tests/toolkit/test_pypcapfile_unit.py
index ad643345ac..ec0021c000 100644
--- a/tests/toolkit/test_pypcapfile_unit.py
+++ b/tests/toolkit/test_pypcapfile_unit.py
@@ -580,7 +580,9 @@ class Record(ctypes.Structure):  # as pcapfile.structs.pcap_packet
     def test_the_captures_warned_of_tcp_over_ipv6_are_bounded(self) -> None:
         from pcapkit.toolkit import pypcapfile as toolkit
 
-        self.assertEqual(toolkit.IPV6_TCP_WARNED_LIMIT, 32)
+        from pcapkit.utilities import config
+
+        self.assertEqual(config.IPV6_TCP_WARNED_LIMIT, 32)
         headers = [[types.SimpleNamespace(ns_resolution=False)] for _ in range(33)]
         with warnings.catch_warnings(record=True) as caught:
             warnings.simplefilter('always')
diff --git a/tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py b/tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py
index 3521067ec0..810dfff316 100644
--- a/tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py
+++ b/tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py
@@ -4,7 +4,7 @@
 The default engine dispatches an IPv4 packet's protocol through the registry
 IPv6 uses, so it reads an AH or IPv6 extension header carried directly in IPv4,
 and the TCP segment or IP-in-IP tunnel behind it, up to
-:data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` headers in a
+:data:`~pcapkit.utilities.config.EXTENSION_HEADER_LIMIT` headers in a
 row (#1606).
 
 A later IPv4 fragment carries no upper-layer header (:rfc:`791#section-3.2`), so
diff --git a/tests/utilities/test_config_unit.py b/tests/utilities/test_config_unit.py
new file mode 100644
index 0000000000..5ff7826c9d
--- /dev/null
+++ b/tests/utilities/test_config_unit.py
@@ -0,0 +1,649 @@
+# -*- coding: utf-8 -*-
+"""Every tunable limit has one home, :mod:`pcapkit.utilities.config`.
+
+GitHub issue #1726: the limits were defined across five packages, and the
+readers in ``ipv6.py`` and ``toolkit/pypcapfile.py`` copied two of them with
+``from ... import``, so patching the defining module left those readers on the
+old value. Now each is a ``PCAPKIT_*`` environment variable read once by
+:mod:`pcapkit.utilities.config`, every reader looks it up there at call time,
+and the old import paths are gone. A bad value warns and falls back to the
+default; ``-1`` lifts a limit, and that or any value above the default warns of
+the risk.
+
+"""
+
+import ast
+import collections
+import contextlib
+import importlib
+import io
+import os
+import pathlib
+import struct
+import subprocess  # nosec: B404
+import sys
+import tempfile
+import unittest
+import warnings
+from typing import TYPE_CHECKING
+from unittest import mock
+
+from tests._support import reimport_once_per_class
+from tests.foundation import _roundtrip as wire
+
+if TYPE_CHECKING:
+    from typing import Any, Iterator
+
+#: Root of the :mod:`pcapkit` package under test.
+PACKAGE = pathlib.Path(__file__).resolve().parents[2] / 'pcapkit'
+
+#: Little-endian PCAP global header: v2.4, snaplen 65535, Ethernet.
+GLOBAL = bytes.fromhex('d4c3b2a1' '0200' '0400' '00000000' '00000000' 'ffff0000' '01000000')
+
+#: Every name that moved, by its old home.
+MOVED = {
+    'pcapkit.protocols.protocol': ['FRAME_LAYER_LIMIT'],
+    'pcapkit.protocols.internet.internet': ['EXTENSION_HEADER_LIMIT'],
+    'pcapkit.protocols.internet.ipv6': ['FRAME_LAYER_LIMIT', 'EXTENSION_HEADER_LIMIT'],
+    'pcapkit.toolkit.pypcapfile': ['IPV6_TCP_WARNED_LIMIT', 'EXTENSION_HEADER_LIMIT'],
+    'pcapkit.dumpkit.common': ['FALLBACK_LIMIT'],
+    'pcapkit.utilities.chardet': ['DETECT_CACHE_SIZE'],
+    'pcapkit.corekit.io': ['_READ_CHUNK'],
+    'pcapkit.corekit.fields.field': ['_MAX_ZERO_PAD_LENGTH', '_MAX_ZERO_PAD_SHORTFALL',
+                                     '_ZERO_PAD_BUDGET_RATIO'],
+    'pcapkit.vendor.default': ['MAX_RETRY', 'CI_MODE'],
+}
+
+#: Every integer variable :mod:`pcapkit.utilities.config` declares, as
+#: ``(name, default, minimum, a phrase of its risk)``, the risk :data:`None` where
+#: ``-1`` is refused. Written out rather than read back, so that a changed name,
+#: default, minimum or risk fails.
+DECLARED = (
+    ('FRAME_LAYER_LIMIT', 64, 0, 'RecursionError (#1610)'),
+    ('EXTENSION_HEADER_LIMIT', 32, 0, 'RecursionError (#1610)'),
+    ('IPV6_TCP_WARNED_LIMIT', 32, 0, 'pins one of its packets in memory'),
+    ('FALLBACK_LIMIT', 10_000, 0, 'a large or endless object graph'),
+    ('DETECT_CACHE_SIZE', 1024, 0, 'never-repeating text'),
+    ('READ_CHUNK', 1_048_576, 1, 'a bogus record length allocates'),
+    ('MAX_ZERO_PAD_LENGTH', 262_144, 0, 'declare a field that long'),
+    ('MAX_ZERO_PAD_SHORTFALL', 65_536, 0, 'without being charged to the running budget'),
+    ('ZERO_PAD_BUDGET_RATIO', 16, 0, 'amplify each octet read'),
+    ('VENDOR_RETRY', 5, 1, None),
+)
+
+
+def options(nxt: int) -> bytes:
+    """An 8-octet Destination Options header: Hdr Ext Len 0 and one PadN."""
+    return bytes([nxt, 0, 1, 4, 0, 0, 0, 0])
+
+
+def chain(count: int) -> bytes:
+    """``count`` Destination Options headers, then a TCP segment."""
+    return b''.join(options(60 if index < count - 1 else 6) for index in range(count)) + wire.tcp(b'x', seq=1)
+
+
+@contextlib.contextmanager
+def environ(**values: str) -> 'Iterator[None]':
+    """Set ``PCAPKIT_`` for each of ``values``, and no other."""
+    with mock.patch.dict(os.environ, {f'PCAPKIT_{name}': value for name, value in values.items()}):
+        yield
+
+
+class ReadingTests(unittest.TestCase):
+    """How a variable's text becomes its value."""
+
+    def setUp(self) -> None:
+        reimport_once_per_class(self)
+        from pcapkit.utilities import config
+        from pcapkit.utilities.warnings import ConfigWarning
+
+        self.config = config
+        self.ConfigWarning = ConfigWarning
+
+    def read(self, text: 'str | None', default: int = 64, **kwargs: object) -> 'tuple[int, list[str]]':
+        """The value of ``PCAPKIT_TEST`` set to ``text``, and what it warned."""
+        values = {} if text is None else {'TEST': text}
+        with environ(**values), warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter('always')
+            value = self.config._read_limit('TEST', default, **kwargs)  # pylint: disable=protected-access
+        self.assertTrue(all(issubclass(item.category, self.ConfigWarning) for item in caught), caught)
+        return value, [str(item.message) for item in caught]
+
+    def test_unset_or_within_the_default_is_taken_silently(self) -> None:
+        self.assertEqual(self.read(None, risk='r'), (64, []))
+        self.assertEqual(self.read('8', risk='r'), (8, []))
+        self.assertEqual(self.read('0', risk='r'), (0, []))
+        self.assertEqual(self.read('64', risk='r'), (64, []))
+
+    def test_a_bad_value_warns_and_falls_back(self) -> None:
+        for text in ('', 'abc', '6.4', '0x40', '-2'):
+            with self.subTest(text=text):
+                value, messages = self.read(text, risk='r')
+                self.assertEqual(value, 64)
+                self.assertEqual(len(messages), 1)
+                self.assertIn('using the default, 64', messages[0])
+
+    def test_minus_one_and_above_the_default_warn_of_the_risk(self) -> None:
+        for text, expected in (('-1', -1), ('65', 65), ('1000', 1000)):
+            with self.subTest(text=text):
+                value, messages = self.read(text, risk='RecursionError (#1610)')
+                self.assertEqual(value, expected)
+                self.assertEqual(len(messages), 1)
+                self.assertIn('RecursionError', messages[0])
+
+    def test_every_message_names_the_variable_and_the_value(self) -> None:
+        for text in ('abc', '-2', '-1', '65'):
+            with self.subTest(text=text):
+                _, messages = self.read(text, risk='r')
+                self.assertEqual(len(messages), 1)
+                self.assertIn(f'PCAPKIT_TEST={text!r}', messages[0])
+
+    def test_without_a_risk_minus_one_is_refused(self) -> None:
+        value, messages = self.read('-1', default=5, minimum=1)
+        self.assertEqual((value, len(messages)), (5, 1))
+        self.assertEqual(self.read('50', default=5, minimum=1), (50, []))
+
+    def test_a_flag(self) -> None:
+        with environ(TEST='Yes'):
+            self.assertIs(self.config._read_flag('TEST'), True)  # pylint: disable=protected-access
+        with environ(TEST='maybe'), warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter('always')
+            self.assertIs(self.config._read_flag('TEST'), False)  # pylint: disable=protected-access
+        self.assertEqual([item.category for item in caught], [self.ConfigWarning])
+
+    def test_both_flag_readers_strip_whitespace(self) -> None:
+        from pcapkit.utilities import logging
+
+        bad = list(logging._BAD_FLAGS)  # pylint: disable=protected-access
+        for text, expected in ((' yes ', True), ('\tOff\n', False)):
+            with self.subTest(text=text), environ(TEST=text), warnings.catch_warnings(record=True) as caught:
+                warnings.simplefilter('always')
+                self.assertIs(self.config._read_flag('TEST'), expected)  # pylint: disable=protected-access
+                self.assertIs(logging._read_flag('PCAPKIT_TEST'), expected)  # pylint: disable=protected-access
+            self.assertEqual(caught, [])
+            self.assertEqual(logging._BAD_FLAGS, bad)  # pylint: disable=protected-access
+
+    def test_the_frame_layer_limit_names_the_recursion_risk(self) -> None:
+        with environ(FRAME_LAYER_LIMIT='-1'), warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter('always')
+            try:
+                importlib.reload(self.config)
+                self.assertEqual(self.config.FRAME_LAYER_LIMIT, -1)
+            finally:
+                os.environ.pop('PCAPKIT_FRAME_LAYER_LIMIT')
+                importlib.reload(self.config)
+        messages = [str(item.message) for item in caught if issubclass(item.category, self.ConfigWarning)]
+        self.assertEqual(len(messages), 1, messages)
+        self.assertIn('PCAPKIT_FRAME_LAYER_LIMIT', messages[0])
+        self.assertIn('RecursionError', messages[0])
+        self.assertEqual(self.config.FRAME_LAYER_LIMIT, 64)
+
+    def test_a_bad_value_does_not_break_the_import(self) -> None:
+        with environ(VENDOR_RETRY='lots', FALLBACK_LIMIT='-7'), \
+                warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter('always')
+            try:
+                importlib.reload(self.config)
+                self.assertEqual((self.config.VENDOR_RETRY, self.config.FALLBACK_LIMIT), (5, 10_000))
+            finally:
+                os.environ.pop('PCAPKIT_VENDOR_RETRY')
+                os.environ.pop('PCAPKIT_FALLBACK_LIMIT')
+                importlib.reload(self.config)
+        self.assertEqual(len([item for item in caught if issubclass(item.category, self.ConfigWarning)]), 2)
+
+
+class DeclarationTests(unittest.TestCase):
+    """Each variable's name, default, minimum and risk, as :data:`DECLARED` writes them."""
+
+    def setUp(self) -> None:
+        reimport_once_per_class(self)
+        from pcapkit.utilities import config
+        from pcapkit.utilities.warnings import ConfigWarning
+
+        self.config = config
+        self.ConfigWarning = ConfigWarning
+
+    def load(self, name: str, text: 'str | None') -> 'tuple[object, list[str]]':
+        """``config.`` with only ``PCAPKIT_`` set, to ``text``, and the ConfigWarnings given."""
+        env = {key: value for key, value in os.environ.items() if not key.startswith('PCAPKIT_')}
+        if text is not None:
+            env[f'PCAPKIT_{name}'] = text
+        try:
+            with mock.patch.dict(os.environ, env, clear=True), warnings.catch_warnings(record=True) as caught:
+                warnings.simplefilter('always')
+                importlib.reload(self.config)
+            return getattr(self.config, name), [str(item.message) for item in caught
+                                                if issubclass(item.category, self.ConfigWarning)]
+        finally:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                importlib.reload(self.config)
+
+    def test_the_table_names_every_variable(self) -> None:
+        names = {row[0] for row in DECLARED} | {'CI_MODE'}
+        self.assertEqual(names, set(self.config.__all__) - {'http_proxy', 'https_proxy'})
+
+    def test_each_variable_is_read_by_its_own_name(self) -> None:
+        for name, default, _, _ in DECLARED:
+            with self.subTest(name=name):
+                self.assertEqual(self.load(name, str(default - 1)), (default - 1, []))
+        with self.subTest(name='CI_MODE'):
+            self.assertEqual(self.load('CI_MODE', 'yes'), (True, []))
+
+    def test_each_default(self) -> None:
+        for name, default, _, _ in DECLARED:
+            with self.subTest(name=name):
+                self.assertEqual(self.load(name, None), (default, []))
+        with self.subTest(name='CI_MODE'):
+            self.assertEqual(self.load('CI_MODE', None), (False, []))
+
+    def test_below_the_minimum_falls_back(self) -> None:
+        for name, default, minimum, risk in DECLARED:
+            # Where -1 means no limit, the first value refused is -2.
+            below = -2 if risk is not None and minimum == 0 else minimum - 1
+            with self.subTest(name=name):
+                self.assertEqual(self.load(name, str(minimum)), (minimum, []))
+                value, messages = self.load(name, str(below))
+                self.assertEqual((value, len(messages)), (default, 1), messages)
+                self.assertIn(f"PCAPKIT_{name}='{below}' is not an integer of at least {minimum}",
+                              messages[0])
+                self.assertIn(f'using the default, {default}', messages[0])
+
+    def test_minus_one_and_above_the_default_warn_of_the_risk(self) -> None:
+        for name, default, _, risk in DECLARED:
+            with self.subTest(name=name):
+                if risk is None:
+                    value, messages = self.load(name, '-1')
+                    self.assertEqual((value, len(messages)), (default, 1), messages)
+                    self.assertEqual(self.load(name, str(default + 1)), (default + 1, []))
+                    continue
+                for text, lifted in (('-1', 'lifts the limit'),
+                                     (str(default + 1), f'is above the default of {default}')):
+                    value, messages = self.load(name, text)
+                    self.assertEqual((value, len(messages)), (int(text), 1), messages)
+                    self.assertIn(f"PCAPKIT_{name}='{text}' {lifted}: ", messages[0])
+                    self.assertIn(risk, messages[0])
+
+
+class LayoutTests(unittest.TestCase):
+    """Where the limits live, and what the module may import."""
+
+    def setUp(self) -> None:
+        reimport_once_per_class(self)
+
+    def imports(self, path: 'pathlib.Path') -> 'set[str]':
+        """Every :mod:`pcapkit` module ``path`` imports, at any level."""
+        found = set()
+        for node in ast.walk(ast.parse(path.read_text(encoding='utf-8'))):
+            if isinstance(node, ast.ImportFrom) and node.module and node.module.startswith('pcapkit'):
+                found.update(f'{node.module}.{alias.name}' if node.module == 'pcapkit.utilities' else node.module
+                             for alias in node.names)
+            elif isinstance(node, ast.Import):
+                found.update(alias.name for alias in node.names if alias.name.startswith('pcapkit'))
+        return found
+
+    def test_config_imports_only_warnings(self) -> None:
+        self.assertEqual(self.imports(PACKAGE / 'utilities' / 'config.py'), {'pcapkit.utilities.warnings'})
+
+    def test_logging_does_not_import_config(self) -> None:
+        self.assertNotIn('pcapkit.utilities.config', self.imports(PACKAGE / 'utilities' / 'logging.py'))
+
+    def test_the_old_paths_are_gone(self) -> None:
+        for module, names in MOVED.items():
+            loaded = importlib.import_module(module)
+            for name in names:
+                with self.subTest(module=module, name=name):
+                    self.assertFalse(hasattr(loaded, name))
+
+    def test_every_variable_is_documented(self) -> None:
+        from pcapkit.utilities import config
+
+        docs = (PACKAGE.parent / 'docs' / 'source' / 'pcapkit' / 'utilities' / 'config.rst').read_text('utf-8')
+        # A limit is named after its variable; a proxy accessor is its lowercase.
+        for name in config.__all__:
+            with self.subTest(name=name):
+                self.assertIn(f'.. envvar:: PCAPKIT_{name.upper()}\n', docs)
+
+
+class ReaderTests(unittest.TestCase):
+    """Patching ``config.`` reaches every reader of it."""
+
+    def setUp(self) -> None:
+        reimport_once_per_class(self)
+        from pcapkit.utilities import config
+
+        self.config = config
+
+    def frame(self, data: bytes) -> 'tuple[object, list[str]]':
+        """The PCAP frame of ``data``, and the messages of the ProtocolWarnings it gave."""
+        from pcapkit.protocols.misc.pcap.frame import Frame
+        from pcapkit.protocols.misc.pcap.header import Header
+        from pcapkit.utilities.warnings import ProtocolWarning
+
+        header = Header(GLOBAL).info
+        record = struct.pack(' None:
+        data = wire.ethernet(wire.ipv4(wire.ipv4(wire.ipv4(wire.tcp(b'x', seq=1), proto=6), proto=4),
+                                       proto=4), 0x0800)
+        with mock.patch.object(self.config, 'FRAME_LAYER_LIMIT', 2):
+            parsed, messages = self.frame(data)
+        self.assertNotIn('TCP', parsed)
+        self.assertEqual(len(messages), 1, messages)
+        self.assertIn('limit of 2 layers', messages[0])
+
+    def test_frame_layer_limit_in_the_ipv6_walk(self) -> None:
+        # Unpatched, IPv6 walks all six headers and the frame limit cuts at the
+        # TCP; patched, the walk itself stops short of the TCP's headers.
+        data = wire.ethernet(wire.ipv6(chain(6), nxt=60), 0x86DD)
+        with mock.patch.object(self.config, 'FRAME_LAYER_LIMIT', 5):
+            parsed, messages = self.frame(data)
+        self.assertNotIn('TCP', parsed)
+        self.assertEqual(len(messages), 1, messages)
+        self.assertIn('limit of 5 layers', messages[0])
+        self.assertIn('from IPv6_Opts on', messages[0])
+
+    def test_extension_header_limit_in_internet_and_the_ipv6_walk(self) -> None:
+        for carrier, data in (('ipv4', wire.ethernet(wire.ipv4(chain(4), proto=60), 0x0800)),
+                              ('ipv6', wire.ethernet(wire.ipv6(chain(4), nxt=60), 0x86DD))):
+            with self.subTest(carrier=carrier):
+                with mock.patch.object(self.config, 'EXTENSION_HEADER_LIMIT', 2):
+                    parsed, messages = self.frame(data)
+                self.assertNotIn('TCP', parsed)
+                self.assertEqual(len(messages), 1, messages)
+                self.assertIn('limit of 2 headers', messages[0])
+
+    def walked(self, parsed: 'Any') -> int:
+        """How many extension headers the frame's IPv6 layer walked itself."""
+        return len(list(parsed['IPv6'].extension_headers.items(multi=True)))
+
+    def test_minus_one_lifts_the_frame_layer_limit_in_protocol(self) -> None:
+        data = wire.ethernet(wire.ipv4(wire.ipv4(wire.ipv4(wire.tcp(b'x', seq=1), proto=6), proto=4),
+                                       proto=4), 0x0800)
+        with mock.patch.object(self.config, 'FRAME_LAYER_LIMIT', -1):
+            parsed, messages = self.frame(data)
+        self.assertIn('TCP', parsed)
+        self.assertEqual(messages, [])
+
+    def test_minus_one_lifts_the_frame_layer_limit_in_the_ipv6_walk(self) -> None:
+        data = wire.ethernet(wire.ipv6(chain(6), nxt=60), 0x86DD)
+        with mock.patch.object(self.config, 'FRAME_LAYER_LIMIT', -1):
+            parsed, messages = self.frame(data)
+        self.assertIn('TCP', parsed)
+        self.assertEqual(messages, [])
+        self.assertEqual(self.walked(parsed), 6)
+
+    def test_minus_one_lifts_the_extension_header_limit(self) -> None:
+        # 40 headers, past the default of 32.
+        for carrier, data in (('ipv4', wire.ethernet(wire.ipv4(chain(40), proto=60), 0x0800)),
+                              ('ipv6', wire.ethernet(wire.ipv6(chain(40), nxt=60), 0x86DD))):
+            with self.subTest(carrier=carrier):
+                parsed, messages = self.frame(data)
+                self.assertNotIn('TCP', parsed)
+                self.assertEqual(len(messages), 1, messages)
+                with mock.patch.object(self.config, 'EXTENSION_HEADER_LIMIT', -1):
+                    parsed, messages = self.frame(data)
+                self.assertIn('TCP', parsed)
+                self.assertEqual(messages, [])
+                if carrier == 'ipv6':
+                    self.assertEqual(self.walked(parsed), 40)
+
+    def test_extension_header_limit_in_pypcapfile(self) -> None:
+        from pcapkit.toolkit import pypcapfile
+
+        walk = pypcapfile._walk_extension_headers  # pylint: disable=protected-access
+        self.assertIsNotNone(walk(60, chain(3)))
+        with mock.patch.object(self.config, 'EXTENSION_HEADER_LIMIT', 2):
+            self.assertIsNone(walk(60, chain(3)))
+        with mock.patch.object(self.config, 'EXTENSION_HEADER_LIMIT', -1):
+            self.assertIsNotNone(walk(60, chain(40)))
+
+    def test_ipv6_tcp_warned_limit_in_pypcapfile(self) -> None:
+        from pcapkit.toolkit import pypcapfile
+
+        warned = pypcapfile._ipv6_tcp_warned  # pylint: disable=protected-access
+        saved = dict(warned)
+        self.addCleanup(lambda: (warned.clear(), warned.update(saved)))
+        for limit, expected in ((1, [2]), (-1, [0, 1, 2])):
+            with self.subTest(limit=limit):
+                warned.clear()
+                keys = iter(range(10))
+                with mock.patch.object(pypcapfile, '_capture_of', lambda packet: (next(keys), None)), \
+                        mock.patch.object(pypcapfile, '_ipv6_carries_tcp', return_value=True), \
+                        mock.patch.object(self.config, 'IPV6_TCP_WARNED_LIMIT', limit), \
+                        warnings.catch_warnings():
+                    warnings.simplefilter('ignore')
+                    for count in range(3):
+                        pypcapfile._decline_ipv6_tcp(None, count)  # type: ignore[arg-type] # pylint: disable=protected-access
+                self.assertEqual(list(warned), expected)
+
+    def test_read_chunk_in_seekable_reader(self) -> None:
+        from pcapkit.corekit.io import SeekableReader
+
+        def asked(limit: int) -> 'list[int]':
+            """What a 30-octet read asks of a 100-octet stream, with ``READ_CHUNK`` patched."""
+            sizes = []  # type: list[int]
+
+            class Stream(io.BytesIO):
+                def read1(self, size: int = -1, /) -> bytes:  # type: ignore[override]
+                    sizes.append(size)
+                    return super().read1(size)
+
+            with mock.patch.object(self.config, 'READ_CHUNK', limit):
+                self.assertEqual(SeekableReader(Stream(bytes(100))).read(30), bytes(30))
+            return sizes
+
+        sizes = asked(7)
+        self.assertTrue(sizes)
+        self.assertLessEqual(max(sizes), 7)
+        # -1 asks for the whole read at once, and only that: ``read(-1)`` would
+        # take the whole stream.
+        self.assertEqual(asked(-1), [30])
+
+    def pad_twice(self, **limits: int) -> None:
+        """Read 10 octets into a 40-octet field twice, in a fresh budget, with ``config.`` patched.
+
+        Each read pads 30 octets. With ``MAX_ZERO_PAD_SHORTFALL=10``, both are
+        charged to the budget, and the second takes it to 60 octets, past the
+        ``50 + 0 * 20`` that ``MAX_ZERO_PAD_LENGTH=50`` and
+        ``ZERO_PAD_BUDGET_RATIO=0`` allow.
+
+        """
+        from pcapkit.corekit.fields import field as module
+        from pcapkit.corekit.fields.strings import BytesField
+
+        field = BytesField(length=40)
+        with contextlib.ExitStack() as stack:
+            for name, value in limits.items():
+                stack.enter_context(mock.patch.object(self.config, name, value))
+            stack.enter_context(module._zero_pad_budget())  # pylint: disable=protected-access
+            for _ in range(2):
+                self.assertEqual(field.unpack(bytes(10), {}), bytes(40))
+
+    def test_max_zero_pad_length_in_field(self) -> None:
+        from pcapkit.corekit.fields import field as field_module
+        from pcapkit.corekit.fields.strings import BytesField
+        from pcapkit.utilities.exceptions import FieldValueError
+
+        field = BytesField(length=100)
+        self.assertEqual(field.unpack(bytes(10), {}), bytes(100))
+        with mock.patch.object(self.config, 'MAX_ZERO_PAD_LENGTH', 50), self.assertRaises(FieldValueError):
+            field.unpack(bytes(10), {})
+
+        # Past the default of 0x40_000, -1 lifts the ceiling, and the budget as
+        # well: the 0x3fff7 octets padded are far past what 10 read would earn.
+        field = BytesField(length=0x40_001)
+        with self.assertRaises(FieldValueError):
+            field.unpack(bytes(10), {})
+        with mock.patch.object(self.config, 'MAX_ZERO_PAD_LENGTH', -1), \
+                field_module._zero_pad_budget():  # pylint: disable=protected-access
+            self.assertEqual(field.unpack(bytes(10), {}), bytes(0x40_001))
+        self.pad_twice(MAX_ZERO_PAD_SHORTFALL=10, MAX_ZERO_PAD_LENGTH=-1, ZERO_PAD_BUDGET_RATIO=0)
+
+    def test_max_zero_pad_shortfall_in_field(self) -> None:
+        from pcapkit.utilities.exceptions import FieldValueError
+
+        self.pad_twice(MAX_ZERO_PAD_LENGTH=50, ZERO_PAD_BUDGET_RATIO=0)
+        with self.assertRaises(FieldValueError):
+            self.pad_twice(MAX_ZERO_PAD_SHORTFALL=10, MAX_ZERO_PAD_LENGTH=50, ZERO_PAD_BUDGET_RATIO=0)
+        self.pad_twice(MAX_ZERO_PAD_SHORTFALL=-1, MAX_ZERO_PAD_LENGTH=50, ZERO_PAD_BUDGET_RATIO=0)
+
+    def test_zero_pad_budget_ratio_in_field(self) -> None:
+        from pcapkit.utilities.exceptions import FieldValueError
+
+        with self.assertRaises(FieldValueError):
+            self.pad_twice(MAX_ZERO_PAD_SHORTFALL=10, MAX_ZERO_PAD_LENGTH=50, ZERO_PAD_BUDGET_RATIO=0)
+        # 50 + 1 * 20 octets read is 70, enough for the 60 padded.
+        self.pad_twice(MAX_ZERO_PAD_SHORTFALL=10, MAX_ZERO_PAD_LENGTH=50, ZERO_PAD_BUDGET_RATIO=1)
+        self.pad_twice(MAX_ZERO_PAD_SHORTFALL=10, MAX_ZERO_PAD_LENGTH=50, ZERO_PAD_BUDGET_RATIO=-1)
+
+    def test_fallback_limit_in_dumpkit(self) -> None:
+        import dictdumper
+
+        from pcapkit.dumpkit.common import make_dumper
+        from pcapkit.utilities.exceptions import UnsupportedCall
+
+        class Node:
+            def __init__(self, **kwargs: object) -> None:
+                self.__dict__.update(kwargs)
+
+        # 2 ** 3 - 1 = 7 objects: within the default, past a patched 3.
+        node = Node(leaf=0)
+        for _ in range(2):
+            node = Node(left=node, right=node)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = os.path.join(tmpdir, 'out.json')
+            make_dumper(dictdumper.JSON)(path)({'node': node}, name='x')
+            with mock.patch.object(self.config, 'FALLBACK_LIMIT', 3), \
+                    self.assertRaisesRegex(UnsupportedCall, 'more than 3 objects'):
+                make_dumper(dictdumper.JSON)(path)({'node': node}, name='x')
+            with mock.patch.object(self.config, 'FALLBACK_LIMIT', -1):
+                make_dumper(dictdumper.JSON)(path)({'node': node}, name='x')
+
+    def fetch(self, retry: int, proxies: 'dict[str, str]') -> 'tuple[list[object], list[str]]':
+        """The proxies of every request a failing crawl makes, and its retry notices.
+
+        ``VENDOR_RETRY`` is patched to ``retry``.
+
+        """
+        import requests
+
+        from pcapkit.vendor import default
+
+        class Crawler(default.Vendor):
+            FLAG = 'isinstance(value, int)'
+            LINK = 'https://example.invalid/registry'
+
+            def count(self, data: object) -> 'collections.Counter[str]':
+                return collections.Counter()
+
+            def process(self, data: object) -> 'tuple[list[str], list[str]]':
+                return [], []
+
+        fetched = []  # type: list[object]
+
+        def fail(url: object, **kwargs: object) -> object:
+            fetched.append(kwargs.get('proxies'))
+            return mock.Mock(ok=False, text='')
+
+        with mock.patch.object(requests, 'get', fail), \
+                mock.patch.object(default, 'get_proxies', return_value=proxies), \
+                mock.patch.object(self.config, 'CI_MODE', True), \
+                mock.patch.object(self.config, 'VENDOR_RETRY', retry), \
+                warnings.catch_warnings(record=True) as caught, \
+                self.assertRaises(requests.RequestException):
+            warnings.simplefilter('always')
+            Crawler.__new__(Crawler)._request()  # pylint: disable=protected-access
+        notices = [str(item.message) for item in caught]
+        return fetched, [text for text in notices if text.startswith((
+            'Connection failed; retry for ', 'Connection failed; retry with proxy for '))]
+
+    def test_vendor_retry_in_vendor(self) -> None:
+        for retry in (2, 3):
+            with self.subTest(retry=retry):
+                fetched, notices = self.fetch(retry, {})
+                self.assertEqual(fetched, [None] * retry)
+                # The notice counts against the configured retry, not the default 5.
+                self.assertEqual(notices, [f'Connection failed; retry for {count}/{retry}...'
+                                           for count in range(1, retry + 1)])
+
+    def test_vendor_retry_in_the_proxy_loop(self) -> None:
+        proxies = {'http': 'http://proxy.invalid'}
+        for retry in (2, 3):
+            with self.subTest(retry=retry):
+                fetched, notices = self.fetch(retry, proxies)
+                self.assertEqual(fetched, [None] * retry + [proxies] * retry)
+                self.assertEqual(notices, [f'Connection failed; retry for {count}/{retry}...'
+                                           for count in range(1, retry + 1)]
+                                 + [f'Connection failed; retry with proxy for {count}/{retry}...'
+                                    for count in range(1, retry + 1)])
+
+    def test_proxies_are_read_at_call_time(self) -> None:
+        from pcapkit.vendor import default
+
+        # Set after the import, as #1726 keeps it: the variables are read on
+        # every call, as get_proxies always read them.
+        with mock.patch.dict(os.environ, {'PCAPKIT_HTTP_PROXY': 'http://late'}):
+            os.environ.pop('PCAPKIT_HTTPS_PROXY', None)
+            self.assertEqual(self.config.http_proxy(), 'http://late')
+            self.assertIsNone(self.config.https_proxy())
+            self.assertEqual(default.get_proxies(), {'http': 'http://late'})
+        with mock.patch.dict(os.environ, {'PCAPKIT_HTTPS_PROXY': 'http://later'}):
+            os.environ.pop('PCAPKIT_HTTP_PROXY', None)
+            self.assertEqual(default.get_proxies(), {'https': 'http://later'})
+
+
+def run(code: str, **values: str) -> 'subprocess.CompletedProcess[str]':
+    """Run ``code`` in a fresh interpreter from the repository root, with ``PCAPKIT_`` set."""
+    env = {key: value for key, value in os.environ.items() if not key.startswith('PCAPKIT_')}
+    env.update({f'PCAPKIT_{name}': value for name, value in values.items()})
+    env['PYTHONPATH'] = os.pathsep.join(filter(None, [str(PACKAGE.parent), env.get('PYTHONPATH')]))
+    return subprocess.run([sys.executable, '-c', code], cwd=PACKAGE.parent, env=env,  # nosec: B603
+                          capture_output=True, text=True, timeout=120, check=False)
+
+
+class ImportTimeTests(unittest.TestCase):
+    """What a variable read at import does, in a fresh interpreter."""
+
+    def test_detect_cache_size_sizes_the_chardet_cache(self) -> None:
+        code = ('import pcapkit.utilities.chardet as c; '
+                'print(c.__file__); print(c.detect.cache_info().maxsize)')
+        # 0 is no cache, not an unbounded one: only -1 lifts the limit.
+        for text, expected in (('7', '7'), ('0', '0'), ('-1', 'None'), (None, '1024')):
+            with self.subTest(text=text):
+                result = run(code, **({} if text is None else {'DETECT_CACHE_SIZE': text}))
+                self.assertEqual(result.returncode, 0, result.stderr)
+                path, size = result.stdout.split()
+                self.assertTrue(path.startswith(str(PACKAGE)), path)
+                self.assertEqual(size, expected)
+
+    def test_a_warning_names_the_importing_module(self) -> None:
+        # Not ``:0``: importlib's frames between the importer and pcapkit
+        # are not counted by warnings.warn, so they are not counted for it.
+        with tempfile.TemporaryDirectory() as tmpdir:
+            pathlib.Path(tmpdir, 'importer_1726.py').write_text('import pcapkit\n', encoding='utf-8')
+            code = ('import sys, warnings; warnings.simplefilter("always"); '
+                    f'sys.path.insert(0, {tmpdir!r}); import importer_1726')
+            result = run(code, FALLBACK_LIMIT='-7', FRAME_LAYER_LIMIT='-1', DEVMODE='maybe')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        for name in ('FALLBACK_LIMIT', 'FRAME_LAYER_LIMIT', 'DEVMODE'):
+            with self.subTest(name=name):
+                self.assertIn(f'importer_1726.py:1: ConfigWarning: PCAPKIT_{name}=', result.stderr)
+        self.assertNotIn('', result.stderr)
+
+    def test_an_error_filter_makes_the_import_raise(self) -> None:
+        code = ('import warnings; warnings.filterwarnings("error", message="PCAPKIT_"); '
+                'import pcapkit')
+        for name, text in (('FALLBACK_LIMIT', '-7'), ('FRAME_LAYER_LIMIT', '-1'), ('DEVMODE', 'maybe')):
+            with self.subTest(name=name):
+                result = run(code, **{name: text})
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn(f"ConfigWarning: PCAPKIT_{name}={text!r}", result.stderr)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/utilities/test_logging.py b/tests/utilities/test_logging.py
index 4daf8e395d..e63be66845 100644
--- a/tests/utilities/test_logging.py
+++ b/tests/utilities/test_logging.py
@@ -5,6 +5,7 @@
 import logging
 import os
 import unittest
+import warnings
 
 from tests._support import load_module, reimport_once_per_class
 
@@ -57,7 +58,10 @@ def test_boolean_environment_flags_are_parsed(self) -> None:
         self.assertTrue(logging_module.VERBOSE)
         self.assertTrue(logging_module.SPHINX_TYPE_CHECKING)
 
-    def test_invalid_boolean_environment_values_fall_back_to_false(self) -> None:
+    def test_invalid_boolean_environment_values_fall_back_to_false_and_warn(self) -> None:
+        # GitHub issue #1726: the value used to be taken as off in silence.
+        # logging loads before warnings, so it records the value and warnings
+        # reports it once it has loaded, naming the variable and the value.
         os.environ['PCAPKIT_DEVMODE'] = 'maybe'
         os.environ['PCAPKIT_VERBOSE'] = 'sometimes'
         os.environ['PCAPKIT_SPHINX'] = 'idk'
@@ -67,6 +71,25 @@ def test_invalid_boolean_environment_values_fall_back_to_false(self) -> None:
         self.assertFalse(logging_module.DEVMODE)
         self.assertFalse(logging_module.VERBOSE)
         self.assertFalse(logging_module.SPHINX_TYPE_CHECKING)
+        self.assertEqual(logging_module._BAD_FLAGS,  # pylint: disable=protected-access
+                         [('PCAPKIT_DEVMODE', 'maybe'), ('PCAPKIT_VERBOSE', 'sometimes'),
+                          ('PCAPKIT_SPHINX', 'idk')])
+
+        load_module('pcapkit.utilities.compat', 'pcapkit/utilities/compat.py')
+        load_module('pcapkit.utilities.exceptions', 'pcapkit/utilities/exceptions.py')
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter('always')
+            warnings_module = load_module('pcapkit.utilities.warnings', 'pcapkit/utilities/warnings.py')
+
+        reported = [str(item.message) for item in caught
+                    if issubclass(item.category, warnings_module.ConfigWarning)]
+        self.assertEqual(len(reported), 3, reported)
+        for message, text in zip(reported, ("PCAPKIT_DEVMODE='maybe'", "PCAPKIT_VERBOSE='sometimes'",
+                                            "PCAPKIT_SPHINX='idk'")):
+            self.assertIn(text, message)
+            self.assertIn('using the default, False', message)
+        # Reported once: the record is drained as it is reported.
+        self.assertEqual(logging_module._BAD_FLAGS, [])  # pylint: disable=protected-access
 
     def test_utilities_package_re_exports_common_helpers(self) -> None:
         package = load_module('pcapkit.utilities', 'pcapkit/utilities/__init__.py')
diff --git a/tests/utilities/test_warning_filters.py b/tests/utilities/test_warning_filters.py
index c19e9e8cff..f697e4c3e8 100644
--- a/tests/utilities/test_warning_filters.py
+++ b/tests/utilities/test_warning_filters.py
@@ -36,7 +36,7 @@
     'FileWarning', 'LayerWarning', 'ProtocolWarning', 'AttributeWarning',
     'DevModeWarning', 'VendorRequestWarning', 'VendorRuntimeWarning',
     'UnknownFieldWarning', 'RegistryWarning', 'SchemaWarning', 'InfoWarning',
-    'SeekWarning', 'ExtractionWarning',
+    'SeekWarning', 'ExtractionWarning', 'ConfigWarning',
     'DPKTWarning', 'ScapyWarning', 'PySharkWarning', 'EmojiWarning',
     'VendorWarning',
     'DeprecatedFormatWarning',
diff --git a/tests/vendor/test_request_prompt_unit.py b/tests/vendor/test_request_prompt_unit.py
index 51f6abe2ca..88b65cf79c 100644
--- a/tests/vendor/test_request_prompt_unit.py
+++ b/tests/vendor/test_request_prompt_unit.py
@@ -376,7 +376,7 @@ def fake_open(link: 'Any' = None, *args: 'Any', **kwargs: 'Any') -> 'bool':
         with contextlib.ExitStack() as stack:
             stack.enter_context(mock.patch.object(self.requests, 'get', fake_get))
             stack.enter_context(mock.patch.object(self.default, 'get_proxies', return_value={}))
-            stack.enter_context(mock.patch.object(self.default, 'CI_MODE', ci_mode))
+            stack.enter_context(mock.patch.object(self.default.config, 'CI_MODE', ci_mode))
             stack.enter_context(mock.patch.object(self.default, 'stdin_is_interactive',
                                                   return_value=interactive))
             stack.enter_context(mock.patch.object(self.default.webbrowser, 'open', fake_open))
diff --git a/tests/vendor/test_user_agent_unit.py b/tests/vendor/test_user_agent_unit.py
index 7725a8d14e..f07c08e41a 100644
--- a/tests/vendor/test_user_agent_unit.py
+++ b/tests/vendor/test_user_agent_unit.py
@@ -7,7 +7,7 @@
 agent outright -- measured 2026-09-19 as HTTP 403 and 126 bytes of robot-policy
 text, against 200 for a descriptive agent, reproduced on four separate articles
 with nothing but the header changing -- so four crawlers could not fetch their
-registry at all, and burned ``MAX_RETRY`` attempts against a refusal that no
+registry at all, and burned ``config.VENDOR_RETRY`` attempts against a refusal that no
 amount of retrying lifts.
 
 .. |requests| replace:: ``requests``