From da00788a467a20811edccf9bd0fd49eb89c9a08f Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Tue, 8 Sep 2026 16:23:24 +0200 Subject: [PATCH 01/17] feat(dpi): refactor DPI rules management --- packages/ns-api/README.md | 434 +++--- packages/ns-api/files/ns.dpi | 149 ++- packages/ns-dpi/Makefile | 2 + packages/ns-dpi/files/20_dpi | 18 - packages/ns-dpi/files/21_dpi_migrate | 13 + packages/ns-dpi/files/connlabel.conf | 3 +- packages/ns-dpi/files/dpi-config | 138 +- packages/ns-dpi/files/dpi-migrate.py | 13 + .../src/nethsec/dpi/__init__.py | 1116 +++++++++++++--- packages/python3-nethsec/tests/test_dpi.py | 1168 +++++++++++------ 10 files changed, 2204 insertions(+), 850 deletions(-) create mode 100755 packages/ns-dpi/files/21_dpi_migrate create mode 100755 packages/ns-dpi/files/dpi-migrate.py diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 382effb66..ad809fa2f 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -3570,7 +3570,10 @@ Data can be limited and paginated by using the `limit` and `page` parameters: api-cli ns.dpi list-applications --data '{"limit": 10, "page": 3}' ``` -**PLEASE NOTE**: `category` field can be missing in some applications/protocols. +**PLEASE NOTE**: + +- `category` field can be missing in some applications/protocols. +- applications and protocols are the ones loaded in memory by the netifyd engine; if the engine can't be queried, applications are read from the installed signatures file. Example response: ```json @@ -3628,255 +3631,350 @@ Example response: } ``` -### list-popular +### list-loaded-applications -List popular applications and protocols: +List the applications the netifyd engine has loaded in memory, as returned by `netifyd --dump-apps`: ```bash -api-cli ns.dpi list-popular +api-cli ns.dpi list-loaded-applications ``` -Data can be limited and paginated by using the `limit` and `page` parameters: +The engine loads only the applications available inside the installed signatures, so the response can be matched +against `list-application-catalog` to find out which applications of the catalog are usable on the machine. -```bash -api-cli ns.dpi list-popular --data '{"limit": 3, "page": 2}' +Example response: +```json +{ + "values": [ + { + "id": 133, + "name": "netify.netflix" + }, + { + "id": 10119, + "name": "netify.adobe" + }, + { + "id": 10552, + "name": "netify.tesla" + } + ] +} ``` -**PLEASE NOTE**: +If the engine can't be queried, the response is: +```json +{ + "error": "applications_not_available" +} +``` -- `category` field can be missing in some applications/protocols. -- `missing` field is true when the application/protocol is not available in the current netifyd database. +### list-loaded-protocols + +List the protocols the netifyd engine has loaded in memory, as returned by `netifyd --dump-protos`: + +```bash +api-cli ns.dpi list-loaded-protocols +``` + +Like `list-loaded-applications`, the response can be matched against `list-protocol-catalog` to find out which +protocols of the catalog are usable on the machine. The engine can also report protocols that are not part of the +catalog, like the `Unknown` pseudo-protocol with id `0`. Example response: +```json +{ + "values": [ + { + "id": 116, + "name": "Warcraft3" + }, + { + "id": 121, + "name": "Dropbox" + }, + { + "id": 130, + "name": "HTTP/Connect" + } + ] +} +``` +If the engine can't be queried, the response is: +```json +{ + "error": "protocols_not_available" +} +``` + +### list-appgroups + +List the application groups, ordered by name: + +```bash +api-cli ns.dpi list-appgroups +``` + +Data can be limited and paginated with the `limit` and `page` parameters, and filtered by name with +`search`: + +```bash +api-cli ns.dpi list-appgroups --data '{"search": "business", "limit": 10, "page": 2}' +``` + +Without `limit` every group is returned, which is what the rule drawer needs to fill its group selector. + +Example response: ```json { "values": { "data": [ { - "id": 10392, - "name": "netify.apple-siri", - "type": "application", - "category": { - "name": "business" - }, - "missing": false - }, - { - "id": 142, - "name": "WhatsApp", - "type": "protocol", - "category": { - "name": "messaging" - }, - "missing": false - }, - { - "name": "whatsapp", - "missing": true - }, - { - "id": 238, - "name": "Apple/Push", - "type": "protocol", - "missing": false + "id": "ns_1a2b3c4d5", + "name": "Business and technology services", + "applications": ["netify.amazon", "netify.apple"], + "application_categories": ["cybersecurity"], + "protocols": ["HTTP/Connect"], + "protocol_categories": ["games"], + "used": true, + "matches": ["dpi/ns_9f8e7d6c5"] } ], "meta": { - "last_page": 3, - "total": 8 + "last_page": 1, + "total": 1 } } } ``` +`used` and `matches` report the rules referencing the group: a group in use can't be deleted. -### list-rules +### add-appgroup -List created rules: +Create an application group: ```bash -api-cli ns.dpi list-rules +api-cli ns.dpi add-appgroup --data '{ + "name": "Business and technology services", + "applications": ["netify.amazon"], + "application_categories": ["cybersecurity"], + "protocols": ["HTTP/Connect"], + "protocol_categories": ["games"] +}' ``` -Example response: +- `name`: mandatory, up to 64 characters, unique among the groups regardless of case +- the four member lists are all optional, but the group must hold at least one member overall +- applications and protocols are named as the **DPI engine** reports them, i.e. the `name` field of + `list-loaded-applications` / `list-loaded-protocols`, **not** the `tag` of the catalogs. For + applications the two coincide (`netify.amazon`); for protocols they don't (`HTTP/Connect` in the + engine, `http-connect` in the catalog) and only the engine name can be matched +- categories are named by tag (`cybersecurity`, `games`), which is the same in the engine and in the + catalogs. Application and protocol categories are distinct vocabularies +Members are validated against what the engine has loaded, with the catalog accepted as well for +applications, so that a group stays editable on a machine whose premium signatures were removed. When +neither source can be read the values are stored unchecked: an unknown value never matches, so refusing +the write would be worse than accepting it. Values are always rejected when empty or when they hold a +character that could break the generated expression. + +Example response: ```json { - "values": [ - { - "config-name": "ns_3869dc35", - "enabled": true, - "device": "eth4", - "interface": "GREEN_1", - "action": "block", - "criteria": [ - { - "id": 156, - "name": "netify.spotify", - "type": "application", - "category": { - "name": "streaming-media" - } - }, - { - "id": 10119, - "name": "netify.adobe", - "type": "application", - "category": { - "name": "business" - } - } - ] - }, - { - "config-name": "ns_f1c6e9e0", - "enabled": false, - "interface": "eth4", - "action": "block", - "criteria": [ - { - "id": 196, - "name": "HTTP/S", - "type": "protocol", - "category": { - "name": "web" - } - } - ] - } - ] + "id": "ns_1a2b3c4d5" } ``` -### add-rule +### edit-appgroup -Add DPI rule: +Same payload as `add-appgroup`, plus the `id` of the group. Lists left out are emptied: ```bash -api-cli ns.dpi add-rule --data '{"enabled": false, "device": "eth4", "applications": [], "protocols": ["HTTP/S"]}' +api-cli ns.dpi edit-appgroup --data '{ + "id": "ns_1a2b3c4d5", + "name": "Business", + "applications": ["netify.amazon", "netify.apple"] +}' ``` -Rundown of required parameters: +Example response: +```json +{ + "id": "ns_1a2b3c4d5" +} +``` -- `enabled`: `true` or `false` -- `device`: device name, e.g. `eth4` -- `applications`: list of application names, e.g. `["netify.spotify", "netify.adobe"]`, refer to `list-applications` - api. -- `protocols`: list of protocol names, e.g. `["HTTP/S"]`, refer to `list-applications` api. +### delete-appgroup -Example response: +Delete an application group: + +```bash +api-cli ns.dpi delete-appgroup --data '{"id": "ns_1a2b3c4d5"}' +``` + +Deleting a group referenced by a rule is refused with the list of the referencing rules: +```json +{ + "validation": { + "errors": [ + { + "parameter": "id", + "message": "appgroup_is_used", + "value": ["dpi/ns_9f8e7d6c5"] + } + ] + } +} +``` +Example response: ```json { "message": "success" } ``` -### delete-rule +### list-rules -Delete DPI rule: +List every rule in priority order, which is the order they are evaluated in: the first rule matching a +flow wins and stops the evaluation. ```bash -api-cli nd.dpi delete-rule --data '{"config-name": "ns_f1c6e9e0"}' +api-cli ns.dpi list-rules ``` -Required parameters: - -- `config-name`: rule name, refer to `list-rules` api. +The list is never paginated nor filtered: it is the whole set, which is also what `order-rules` needs. +Rules hidden with `ns_visible '0'` are the only ones left out. `managed` tells whether the rule was +created through the API: an unmanaged rule can be renamed, enabled, reordered and deleted, but not +edited, and it carries the raw `criteria` it matches on. Example response: ```json { - "message": "success" + "values": [ + { + "id": "ns_3869dc35", + "name": "Block streaming", + "enabled": true, + "action": "block", + "source": ["192.168.1.0/24"], + "appgroups": [ + { + "id": "ns_1a2b3c4d5", + "name": "Streaming services" + } + ], + "managed": true, + "index": 0 + }, + { + "id": "ns_f1c6e9e0", + "name": "Legacy rule", + "enabled": true, + "action": "block", + "source": [], + "appgroups": [], + "managed": false, + "criteria": "local_ip == 192.168.100.22 && app == 'netify.facebook';", + "index": 1 + } + ] } ``` -### edit-rule +### add-rule -Edit DPI rule: +Add a DPI rule: ```bash -api-cli ns.dpi edit-rule --data '{"config-name": "ns_f1c6e9e0", "enabled": true, "device": "eth4", "applications": ["netify.spotify", "netify.adobe"], "protocols": []}' +api-cli ns.dpi add-rule --data '{ + "name": "Block streaming", + "enabled": true, + "action": "block", + "source": ["192.168.1.0/24"], + "appgroups": ["ns_1a2b3c4d5"], + "position": "bottom" +}' ``` -Rundown of required parameters: -- `config-name`: rule name, refer to `list-rules` api. +Parameters: + +- `name`: mandatory, up to 64 characters - `enabled`: `true` or `false` -- `device`: device name, e.g. `eth4` -- `applications`: list of application names, e.g. `["netify.spotify", "netify.adobe"]`, refer to `list-applications` - api. -- `protocols`: list of protocol names, e.g. `["HTTP/S"]`, refer to `list-applications` api. +- `action`: `block` or `allow`. `allow` blocks nothing: it labels the flow and, since every rule halts + on match, stops the evaluation before any block rule below it can fire +- `source`: list of addresses, networks or ranges, IPv4 and IPv6 alike, e.g. + `["192.168.1.1", "192.168.1.0/24", "10.0.0.10-10.0.0.20"]`. An empty list matches every host. Ranges + are expanded to CIDR blocks when the rule is generated +- `appgroups`: config names of the application groups the rule matches, refer to `list-appgroups`. **At + least one is required**: a rule with a source and no group would be an IP-level rule, which the + firewall does better, and a rule with neither would match nothing +- `position`: `top` to evaluate the rule before every other one, `bottom` (the default) after them + +There is no device or interface parameter: per-interface rules are not reachable from the API. Example response: ```json { - "message": "success" + "id": "ns_3869dc35" } ``` -### list-devices +### edit-rule -List available devices to be added to DPI rules: +Same payload as `add-rule` minus `position`, plus the `id` of the rule. The priority is left untouched, +use `order-rules` to move the rule. ```bash -api-cli ns.dpi list-devices +api-cli ns.dpi edit-rule --data '{ + "id": "ns_3869dc35", + "name": "Block streaming", + "enabled": false, + "action": "block", + "source": [], + "appgroups": ["ns_1a2b3c4d5"] +}' ``` +Only managed rules can be edited: editing a rule carrying a hand-written criteria is refused with +`rule_not_managed`, because it has no source and no group to rebuild it from. + Example response: ```json { - "values": [ - { - "interface": "GREEN_1", - "device": "eth0" - }, - { - "interface": "GREEN_2", - "device": "eth4" - } - ] + "id": "ns_3869dc35" } ``` -### list-exemptions +### delete-rule -List configured global exemptions: +Delete a DPI rule. The priorities of the remaining rules are renumbered, so no gap is left behind: ```bash -api-cli ns.dpi list-exemptions +api-cli ns.dpi delete-rule --data '{"id": "ns_f1c6e9e0"}' ``` Example response: + ```json { - "values": [ - { - "config-name": "cfg024ffe", - "enabled": true, - "criteria": "192.168.1.1", - "description": "my ex" - } - ] + "message": "success" } ``` -### add-exemption -Add global exemption: +### rename-rule + +Rename a rule, managed or not: ```bash -api-cli ns.dpi add-rule --data '{"criteria": "192.168.1.1", "description": "my host", "enabled": true}' +api-cli ns.dpi rename-rule --data '{"id": "ns_f1c6e9e0", "name": "Legacy rule"}' ``` -Rundown of required parameters: - -- `enabled`: `true` or `false` -- `critera`: an IP address like `192.168.1.1` -- `description`: an optional description - Example response: ```json @@ -3885,34 +3983,30 @@ Example response: } ``` -It can raise a validation error if the criteria is duplicated. Example: +### enable-rule + +Enable a rule, managed or not: + +```bash +api-cli ns.dpi enable-rule --data '{"id": "ns_f1c6e9e0"}' +``` + +Example response: ```json { - "validation": { - "errors": [ - { - "parameter": "criteria", - "message": "criteria_already_exists", - "value": "192.168.1.3" - } - ] - } + "message": "success" } ``` -### delete-exemption +### disable-rule -Delete global exemption rule: +Disable a rule, managed or not: ```bash -api-cli nd.dpi delete-exemption --data '{"config-name": "ns_f1c6e9e0"}' +api-cli ns.dpi disable-rule --data '{"id": "ns_f1c6e9e0"}' ``` -Required parameters: - -- `config-name`: exemption name, refer to `list-exemptions` api. - Example response: ```json @@ -3921,25 +4015,23 @@ Example response: } ``` -### edit-rule +### order-rules -Edit global exemption: +Reorder the rules, renumbering their priorities from the given order: ```bash -api-cli ns.dpi edit-rule --data '{"config-name": "ns_f1c6e9e0", "criteria": "192.168.1.1", "description": "my host", "enabled": true}' +api-cli ns.dpi order-rules --data '{"order": ["ns_f1c6e9e0", "ns_3869dc35"]}' ``` -Rundown of required parameters: -- `config-name`: rule name, refer to `list-rules` api. -- `enabled`: `true` or `false` -- `critera`: an IP address like `192.168.1.1` -- `description`: an optional description +The order must name **every** rule exactly once, hidden ones included, as `list-rules` returns them. A +partial order is refused with `invalid_order`: silently moving the rules left out is never what the +caller meant. Example response: ```json { - "message": "success" + "values": ["ns_f1c6e9e0", "ns_3869dc35"] } ``` diff --git a/packages/ns-api/files/ns.dpi b/packages/ns-api/files/ns.dpi index 84c3dadc5..cac3b9202 100755 --- a/packages/ns-api/files/ns.dpi +++ b/packages/ns-api/files/ns.dpi @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2023 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -22,45 +22,66 @@ if cmd == 'list': }, 'list-rules': {}, 'add-rule': { + 'name': 'str', 'enabled': False, - 'device': 'str', - 'applications': 'str', - 'protocols': 'str' - }, - 'delete-rule': { - 'config-name': 'str' + 'action': 'str', + 'source': [], + 'appgroups': [], + 'position': 'str' }, 'edit-rule': { - 'config-name': 'str', + 'id': 'str', + 'name': 'str', 'enabled': False, - 'device': 'str', - 'applications': 'str', - 'protocols': 'str' + 'action': 'str', + 'source': [], + 'appgroups': [] }, - 'add-exemption': { - 'criteria': 'str', - 'description': 'str', - 'enabled': False + 'delete-rule': { + 'id': 'str' }, - 'delete-exemption': { - 'config-name': 'str' + 'rename-rule': { + 'id': 'str', + 'name': 'str' }, - 'edit-exemption': { - 'config-name': 'str', - 'criteria': 'str', - 'description': 'str', - 'enabled': False + 'enable-rule': { + 'id': 'str' }, - 'list-exemptions': {}, - 'list-devices': {}, - 'list-popular': { - 'limit': 32, - 'page': 32 + 'disable-rule': { + 'id': 'str' + }, + 'order-rules': { + 'order': ['ns_1234', 'ns_5678'] }, 'list-application-categories': {}, 'list-application-catalog': {}, + 'list-loaded-applications': {}, 'list-protocol-categories': {}, - 'list-protocol-catalog': {} + 'list-protocol-catalog': {}, + 'list-loaded-protocols': {}, + 'list-appgroups': { + 'search': 'str', + 'limit': 32, + 'page': 32 + }, + 'add-appgroup': { + 'name': 'str', + 'applications': [], + 'application_categories': [], + 'protocols': [], + 'protocol_categories': [] + }, + 'edit-appgroup': { + 'id': 'str', + 'name': 'str', + 'applications': [], + 'application_categories': [], + 'protocols': [], + 'protocol_categories': [] + }, + 'delete-appgroup': { + 'id': 'str' + } })) elif cmd == 'call': action = sys.argv[2] @@ -80,6 +101,14 @@ elif cmd == 'call': print(json.dumps({'values': content})) except Exception: print(json.dumps({})) + elif action == 'list-loaded-applications': + try: + applications = dpi.load_applications() + except Exception: + print(json.dumps(utils.generic_error('applications_not_available'))) + else: + print(json.dumps({'values': [{'id': app_id, 'name': app_name} + for app_id, app_name in applications.items()]})) elif action == 'list-protocol-categories': try: with open('/etc/netifyd/netify-protocol-categories.json', 'r') as f: @@ -94,6 +123,14 @@ elif cmd == 'call': print(json.dumps({'values': content})) except Exception: print(json.dumps({})) + elif action == 'list-loaded-protocols': + try: + protocols = dpi.load_protocols() + except Exception: + print(json.dumps(utils.generic_error('protocols_not_available'))) + else: + print(json.dumps({'values': [{'id': proto_id, 'name': proto_name} + for proto_id, proto_name in protocols.items()]})) elif action == 'list-applications': data = json.JSONDecoder().decode(sys.stdin.read()) result = dpi.list_applications(data.get('search', None), data.get('limit', None), data.get('page', 1)) @@ -102,37 +139,55 @@ elif cmd == 'call': print(json.dumps({'values': dpi.list_rules(e_uci)})) elif action == 'add-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.add_rule(e_uci, data['enabled'], data['device'], 'block', data['applications'], data['protocols']) - print(json.dumps({'message': 'success'})) + config_name = dpi.add_rule(e_uci, data['name'], data['enabled'], data['action'], + data.get('source', []), data.get('appgroups', []), + data.get('position', 'bottom')) + print(json.dumps({'id': config_name})) + elif action == 'edit-rule': + data = json.JSONDecoder().decode(sys.stdin.read()) + config_name = dpi.edit_rule(e_uci, data['id'], data['name'], data['enabled'], data['action'], + data.get('source', []), data.get('appgroups', [])) + print(json.dumps({'id': config_name})) elif action == 'delete-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.delete_rule(e_uci, data['config-name']) + dpi.delete_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) - elif action == 'edit-rule': + elif action == 'rename-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.edit_rule(e_uci, data['config-name'], data['enabled'], data['device'], 'block', data['applications'], - data['protocols']) + dpi.rename_rule(e_uci, data['id'], data['name']) print(json.dumps({'message': 'success'})) - elif action == 'add-exemption': + elif action == 'enable-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.add_exemption(e_uci, data['criteria'], data['description'], data['enabled']) + dpi.enable_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) - elif action == 'delete-exemption': + elif action == 'disable-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.delete_exemption(e_uci, data['config-name']) + dpi.disable_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) - elif action == 'edit-exemption': + elif action == 'order-rules': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.edit_exemption(e_uci, data['config-name'], data['criteria'], data['description'], data['enabled']) - print(json.dumps({'message': 'success'})) - elif action == 'list-exemptions': - print(json.dumps({'values': dpi.list_exemptions(e_uci)})) - elif action == 'list-devices': - print(json.dumps({'values': dpi.list_devices(e_uci)})) - elif action == 'list-popular': + print(json.dumps({'values': dpi.order_rules(e_uci, data['order'])})) + elif action == 'list-appgroups': data = json.JSONDecoder().decode(sys.stdin.read()) - result = dpi.list_popular(e_uci, data.get('limit', None), data.get('page', 1)) + result = dpi.list_appgroups(e_uci, data.get('search', None), data.get('limit', None), + data.get('page', 1)) print(json.dumps({'values': result})) + elif action == 'add-appgroup': + data = json.JSONDecoder().decode(sys.stdin.read()) + config_name = dpi.add_appgroup(e_uci, data['name'], data.get('applications', []), + data.get('application_categories', []), data.get('protocols', []), + data.get('protocol_categories', [])) + print(json.dumps({'id': config_name})) + elif action == 'edit-appgroup': + data = json.JSONDecoder().decode(sys.stdin.read()) + config_name = dpi.edit_appgroup(e_uci, data['id'], data['name'], data.get('applications', []), + data.get('application_categories', []), data.get('protocols', []), + data.get('protocol_categories', [])) + print(json.dumps({'id': config_name})) + elif action == 'delete-appgroup': + data = json.JSONDecoder().decode(sys.stdin.read()) + dpi.delete_appgroup(e_uci, data['id']) + print(json.dumps({'message': 'success'})) except KeyError as e: print(json.dumps(utils.validation_error(e.args[0], 'required'))) except json.JSONDecodeError: diff --git a/packages/ns-dpi/Makefile b/packages/ns-dpi/Makefile index dc28ddbb3..17af07c6b 100644 --- a/packages/ns-dpi/Makefile +++ b/packages/ns-dpi/Makefile @@ -78,7 +78,9 @@ define Package/ns-dpi/install $(INSTALL_BIN) ./files/dpi-license-update.py $(1)/usr/sbin/dpi-license-update $(INSTALL_BIN) ./files/dpi-data-update.py $(1)/usr/sbin/dpi-data-update $(INSTALL_BIN) ./files/dpi-update.py $(1)/usr/sbin/dpi-update + $(INSTALL_BIN) ./files/dpi-migrate.py $(1)/usr/sbin/dpi-migrate $(INSTALL_CONF) ./files/20_dpi $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/21_dpi_migrate $(1)/etc/uci-defaults/21_dpi_migrate $(INSTALL_BIN) ./files/99-dpi-license-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-license-update-cron $(INSTALL_BIN) ./files/99-dpi-data-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-data-update-cron $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/register diff --git a/packages/ns-dpi/files/20_dpi b/packages/ns-dpi/files/20_dpi index f5e6adaf2..da9094280 100644 --- a/packages/ns-dpi/files/20_dpi +++ b/packages/ns-dpi/files/20_dpi @@ -6,24 +6,6 @@ config main 'config' option log_blocked '0' option enabled '0' option firewall_exemption '0' - list popular_filters 'netify.facebook' - list popular_filters 'netify.amazon-prime' - list popular_filters 'netify.whatsapp' - list popular_filters 'netify.instagram' - list popular_filters 'netify.netflix' - list popular_filters 'netify.telegram' - list popular_filters 'netify.tiktok' - list popular_filters 'netify.youtube' - list popular_filters 'netify.facebook-messenger' - list popular_filters 'netify.twitter' - list popular_filters 'netify.vimeo' - list popular_filters 'netify.snapchat' - list popular_filters 'netify.pinterest' - list popular_filters 'netify.nordvpn' - list popular_filters 'DoT' - list popular_filters 'netify.twitch' - list popular_filters 'netify.teamviewer' - list popular_filters 'DoH' EOI fi diff --git a/packages/ns-dpi/files/21_dpi_migrate b/packages/ns-dpi/files/21_dpi_migrate new file mode 100755 index 000000000..4521e6801 --- /dev/null +++ b/packages/ns-dpi/files/21_dpi_migrate @@ -0,0 +1,13 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Migrate /etc/config/dpi from the schema used before application groups existed. Safe to run on every +# boot: dpi-migrate is a no-op once there is nothing left in the old schema. + +set -e + +/usr/sbin/dpi-migrate diff --git a/packages/ns-dpi/files/connlabel.conf b/packages/ns-dpi/files/connlabel.conf index 784b84ec4..6131ac74c 100644 --- a/packages/ns-dpi/files/connlabel.conf +++ b/packages/ns-dpi/files/connlabel.conf @@ -4,4 +4,5 @@ 3 bulk 4 best_effort 5 video -6 voice \ No newline at end of file +6 voice +7 netify-allowed \ No newline at end of file diff --git a/packages/ns-dpi/files/dpi-config b/packages/ns-dpi/files/dpi-config index 969796d0a..1c114cd08 100755 --- a/packages/ns-dpi/files/dpi-config +++ b/packages/ns-dpi/files/dpi-config @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -9,7 +9,8 @@ from euci import EUci import json import subprocess import os -from nethsec import utils, objects +import syslog +from nethsec import utils, objects, dpi def get_interface_ips(interface): ret = list() @@ -50,6 +51,13 @@ config = { "path": "/var/run/netifyd/", "prefix": "dpi-actions", }, + "allow": { + "target_type": "ctlabel", + "target_enabled": True, + "labels": [ + "netify-allowed" + ] + }, "analyzed": { "target_type": "ctlabel", "target_enabled": True, @@ -58,7 +66,18 @@ config = { ] } }, - "actions": {}, + # analyzed comes first and lets the evaluation continue, so the label is never lost to a rule + # that halts on match + "actions": { + "analyzed": { + "enabled": True, + "priority": 0, + "halt_on_match": False, + "criteria": 'detection_guessed || detection_complete;', + "targets": ["analyzed"], + "exemptions": [] + } + }, "exemptions": [] } @@ -86,77 +105,86 @@ for ex_name in exemptions: rcount = 0 valid_actions = ['block', 'bulk', 'best_effort', 'video', 'voice'] +legacy_fields = ('device', 'application', 'protocol', 'category') for section in u.get_all('dpi'): if u.get('dpi', section) != 'rule': continue rule = u.get_all('dpi', section) - if rule['action'] not in valid_actions or not rule['enabled']: + if not rule.get('enabled'): continue - device = rule.get('device', '*') - - if 'criteria' in rule: - # criteria has precedence over sources, protocol, category and application - criteria = rule['criteria'].replace('"',"'") - else: - # generate criteria from source, protocol, category and application - sources = [] - for source in rule.get('source', []): - if objects.is_object_id(source): - for ip in objects.get_object_ips(u, source): - sources.append(f'local_ip == {ip}') - else: # custom local IPs - sources.append(f'local_ip == {source}') - - applications = [] - for app in rule.get('application', []): - applications.append(f"app == '{app}'") - for proto in rule.get('protocol', []): - applications.append(f"proto == '{proto.lower()}'") - for cat in rule.get('category', []): - applications.append(f"category == '{cat.lower()}'") - - sources_s = ' or '.join(sources) - applications_s = ' or '.join(applications) - criteria = f'(iface_nfq_src == \'{device}\' or iface_nfq_dst == \'{device}\') && ' - if len(sources) < 1: - criteria += f'({applications_s}) ;' - elif len(applications) < 1: - criteria += f'({sources_s}) ;' - else: - criteria += f'({sources_s}) && ({applications_s}) ;' + # a rule from the schema used before application groups existed carries at least one of these + # fields; their presence is a permanent marker, unrelated to whether the migration has since given + # the section a priority + is_legacy = any(field in rule for field in legacy_fields) + + if not is_legacy and 'criteria' not in rule: + # current schema: application groups, optionally narrowed to a source, or a rule migrated from + # a global exemption (source only, no group — dpi.migrate_schema) + if rule.get('action') not in dpi.DPI_RULE_ACTIONS: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, unknown action {rule.get('action')}") + continue + criteria = dpi.build_rule_criteria(u, rule, require_appgroup=False) + if not criteria: + # never emit an empty criteria: it is a parse error, and a parse error kills the agent + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, it matches nothing") + continue + try: + priority = int(rule.get('priority', 1)) + except ValueError: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, invalid priority") + continue + config["actions"][section] = { + "enabled": rule['enabled'] == '1', + "priority": priority, + "halt_on_match": True, + "criteria": criteria, + "targets": [rule['action']], + "exemptions": [] + } + continue - vlan_id = None - base_if = None - for item in utils.get_all_by_type(u, 'network', 'device').values(): - if item.get('vid', None) is not None and item.get('name', '') == device: - vlan_id = item.get('vid', None) - base_if = item.get('ifname', None) - break + if not is_legacy: + # raw criteria: hand-written, or a rule migrated from the old schema with its behaviour frozen + # verbatim (dpi.migrate_schema) — halts on match and keeps its own priority, same as any other + # rule of the current schema + if rule.get('action') not in dpi.DPI_RULE_ACTIONS: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, unknown action {rule.get('action')}") + continue + try: + priority = int(rule.get('priority', 1)) + except ValueError: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, invalid priority") + continue + config["actions"][section] = { + "enabled": rule['enabled'] == '1', + "priority": priority, + "halt_on_match": True, + "criteria": dpi.freeze_legacy_criteria(u, rule), + "targets": [rule['action']], + "exemptions": [] + } + continue - if vlan_id is not None and base_if is not None: - criteria = f'vlan_id == {vlan_id} && {criteria}' - device = base_if + # legacy rule, still on the old schema: emitted as it was until the migration converts it + if rule.get('action') not in valid_actions: + continue targets = [rule['action']] if 'log' in rule and rule['log'] == '1': targets.append('log') config["actions"][f"rule{rcount}"] = { "enabled": rule['enabled'] == '1', - "criteria": criteria, + # legacy rules never halt, so they are evaluated after the ones of the current schema and + # their relative order does not matter + "priority": 1000 + rcount, + "halt_on_match": False, + "criteria": dpi.freeze_legacy_criteria(u, rule), "targets": targets, "exemptions": rule.get('exemption', []) } rcount = rcount + 1 -# add global analyzed for all flows that are not blocked -config["actions"]["analyzed"] = { - "enabled": True, - "criteria": 'detection_guessed || detection_complete;', - "targets": ["analyzed"], - "exemptions": [] -} - # if directories do not exist, create them if not os.path.exists(os.path.dirname(cfg_file)): os.makedirs(os.path.dirname(cfg_file)) diff --git a/packages/ns-dpi/files/dpi-migrate.py b/packages/ns-dpi/files/dpi-migrate.py new file mode 100755 index 000000000..1a3b32d15 --- /dev/null +++ b/packages/ns-dpi/files/dpi-migrate.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# this script is supposed to be run by the 21_dpi_migrate uci-defaults + +from euci import EUci +from nethsec import dpi + +dpi.migrate_schema(EUci()) diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py index 5de35668e..57cc50cd5 100644 --- a/packages/python3-nethsec/src/nethsec/dpi/__init__.py +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2023 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -9,14 +9,15 @@ Library that handles the DPI rules. """ +import ipaddress import json import subprocess -from fnmatch import fnmatch +import syslog import math from euci import EUci -from nethsec import utils, firewall +from nethsec import objects, utils from nethsec.utils import ValidationError @@ -36,6 +37,32 @@ def __load_applications() -> dict[int, str]: return applications +def load_applications() -> dict[int, str]: + """ + Reads the applications loaded in memory by the engine, using the netifyd --dump-apps command. + + Returns: + dict of applications, the key is the application id and the value is the application name + """ + result = subprocess.run(['netifyd', '--dump-apps'], check=True, capture_output=True) + applications = dict[int, str]() + for line in result.stdout.decode().splitlines(): + # lines can be empty + if len(line.strip()) < 1: + continue + line_split = line.split(":", 1) + if len(line_split) < 2: + continue + try: + application_id = int(line_split[0].strip()) + except ValueError: + # skip lines not in the "id: name" format, like headers + continue + applications[application_id] = line_split[1].strip() + + return applications + + def __load_application_categories() -> dict[int, dict[str]]: """ Reads the application categories from the netify-categories.json file. @@ -130,7 +157,11 @@ def __load_blocklist() -> list[dict[str]]: list of dicts, each dict contains the property "id", "name", "type" and "category" """ result = list[dict[str]]() - applications = __load_applications() + try: + applications = load_applications() + except Exception: + # the engine can't be queried, fall back to the signatures file + applications = __load_applications() application_categories = __load_application_categories() for application_id, application_name in applications.items(): @@ -159,29 +190,6 @@ def __load_blocklist() -> list[dict[str]]: return result -def list_devices(e_uci: EUci): - """ - List device-interface available for filtering. - - Returns: - list of dicts, each dict contains the property "interface" and "device" - """ - instance_name = list(e_uci.get('netifyd').keys())[0] - devices = e_uci.get('netifyd', instance_name, 'internal_if', default=[], list=True) - for zone in firewall.list_zones(e_uci).values(): - if zone['name'] == 'wan': - continue - network_devices = utils.get_all_devices_by_zone(e_uci, zone['name']) - devices = list(set(list(devices) + network_devices)) - ret = [] - for item in devices: - interface_name = utils.get_interface_from_device(e_uci, item) - ret.append({ - 'interface': interface_name if interface_name is not None else item, - 'device': item - }) - return ret - def list_applications(search: str = None, limit: int = None, page: int = 1) -> dict: """ List applications available for filtering. @@ -221,40 +229,297 @@ def list_applications(search: str = None, limit: int = None, page: int = 1) -> d } -def list_popular(e_uci: EUci, limit: int = None, page: int = 1) -> dict: +def __apply(e_uci: EUci): + """ + Commit the dpi config and reload the dpi service. + + Used by the schema migration only: it runs at boot with nobody around to confirm the pending uci + changes, so it must take effect on its own. Rule and appgroup CRUD go through the normal pending + uci changes instead, applied later by ns.commit like every other section. + """ + e_uci.commit('dpi') + subprocess.run(["/etc/init.d/dpi", "reload"], check=True) + + +def __toggle_engine(e_uci: EUci): + count_enabled = 0 + for section in e_uci.get_all('dpi'): + if e_uci.get('dpi', section, default="") == "rule" and e_uci.get('dpi', section, 'enabled', default="0") == "1": + count_enabled = count_enabled + 1 + + if count_enabled > 0: + e_uci.set('dpi', 'config', 'enabled', '1') + else: + e_uci.set('dpi', 'config', 'enabled', '0') + + +# Application groups: a NethSecurity abstraction, the netifyd plugin has no group primitive. +# A group is a named set of match members, expanded into an inline expression at generation time. + +NETIFYD_DATA_DIR = '/etc/netifyd' + +# UCI list name of every member kind, mapped to the API parameter it is fed by +APPGROUP_MEMBERS = { + 'app': 'applications', + 'app_category': 'application_categories', + 'proto': 'protocols', + 'proto_category': 'protocol_categories' +} + +# validation error message of every member kind +__APPGROUP_MEMBER_ERRORS = { + 'app': 'invalid_application', + 'app_category': 'invalid_application_category', + 'proto': 'invalid_protocol', + 'proto_category': 'invalid_protocol_category' +} + +# characters that would let a member value escape its quoted literal, or terminate the expression +__APPGROUP_FORBIDDEN_CHARS = '\'"\\;()\n\r\t' + +APPGROUP_NAME_MAX_LENGTH = 64 + + +def load_categories() -> dict[str, dict[int, str]]: + """ + Reads the categories loaded in memory by the engine, using the netifyd --dump-categories command. + + The dump reports one category per line, as ": : ". + + Returns: + dict of categories indexed by type, e.g. `{'application': {1: 'adult'}, 'protocol': {2: 'database'}}`. + Ids are the ones the agent uses internally: they do not match the ids of the downloaded catalogs, + so categories must always be crossed by tag. + """ + result = subprocess.run(['netifyd', '--dump-categories'], check=True, capture_output=True) + categories = dict[str, dict[int, str]]() + for line in result.stdout.decode().splitlines(): + line_split = line.split(":", 2) + if len(line_split) < 3: + continue + try: + category_id = int(line_split[0].strip()) + except ValueError: + # skip lines not in the "id: type: tag" format, like headers + continue + category_type = line_split[1].strip() + categories.setdefault(category_type, dict[int, str]())[category_id] = line_split[2].strip() + + return categories + + +def __load_catalog_tags(filename: str) -> set[str]: """ - List popular applications available for filtering. + Reads the tags of a downloaded catalog, one of the `/etc/netifyd/netify-*.json` files. Args: - - limit: limit the number of results - - page: page number + - filename: name of the catalog file inside the netifyd data directory Returns: - list of dicts, each dict contains the property "id", "name", "type" and "category" + set of the tags the catalog lists """ - popular_filters = e_uci.get('dpi', 'config', 'popular_filters', default=[], list=True) - block_list = {block['name']: block for block in __load_blocklist()} - result = [] + with open(f'{NETIFYD_DATA_DIR}/{filename}', 'r') as file: + return {entry['tag'] for entry in json.load(file) if entry.get('tag')} + + +def __appgroup_vocabulary(kind: str) -> set[str] | None: + """ + Values a member of the given kind can take, lowercased. + + The engine is the authority: only what it has loaded can be matched. The downloaded catalog backs + applications up, so that a group stays editable on a machine that lost its premium signatures. + + Args: + - kind: one of the keys of `APPGROUP_MEMBERS` - for popular_filter in popular_filters: - if popular_filter in block_list.keys(): - result.append(block_list[popular_filter] | {'missing': False}) + Returns: + set of accepted values, or None when no source is available and the values cannot be checked + """ + vocabulary = set[str]() + try: + if kind == 'app': + vocabulary |= {value.lower() for value in load_applications().values()} + elif kind == 'proto': + vocabulary |= {value.lower() for value in load_protocols().values()} else: - result.append({ - 'name': popular_filter, - 'missing': True - }) + category_type = 'application' if kind == 'app_category' else 'protocol' + vocabulary |= {value.lower() for value in load_categories().get(category_type, {}).values()} + except Exception: + # the engine can't be queried, fall back to the downloaded catalogs below + pass + + if kind != 'proto': + # protocol names come from the agent itself and have no counterpart in the catalog, where the + # same protocol is listed under a different tag: the catalog would accept unmatchable values + catalogs = { + 'app': 'netify-application-catalog.json', + 'app_category': 'netify-application-categories.json', + 'proto_category': 'netify-protocol-categories.json' + } + try: + vocabulary |= {value.lower() for value in __load_catalog_tags(catalogs[kind])} + except Exception: + pass - total = len(result) + # an unknown value never matches, so a missing vocabulary is a reason to skip the check, not to refuse + return vocabulary if vocabulary else None + + +def __validate_appgroup_members(kind: str, values: list[str]) -> list[str]: + """ + Validate the members of one kind, stripping and deduplicating them. + + Args: + - kind: one of the keys of `APPGROUP_MEMBERS` + - values: list of member values + + Returns: + the sanitized list of members + + Raises: + - ValidationError: if a member is empty, holds a character that would break the expression, or is + unknown to both the engine and the catalog + """ + parameter = APPGROUP_MEMBERS[kind] + message = __APPGROUP_MEMBER_ERRORS[kind] + if not values: + return [] + + vocabulary = __appgroup_vocabulary(kind) + members = [] + seen = set[str]() + for value in values: + if not isinstance(value, str): + raise ValidationError(parameter, message, value) + member = value.strip() + if not member or any(char in member for char in __APPGROUP_FORBIDDEN_CHARS): + raise ValidationError(parameter, message, value) + if vocabulary is not None and member.lower() not in vocabulary: + raise ValidationError(parameter, message, member) + if member.lower() not in seen: + seen.add(member.lower()) + members.append(member) + + return members + + +def __validate_appgroup(e_uci: EUci, name: str, applications: list[str], application_categories: list[str], + protocols: list[str], protocol_categories: list[str], + config_name: str = None) -> tuple[str, dict[str, list[str]]]: + """ + Validate a group before it is stored. + + Args: + - e_uci: euci instance + - name: name of the group + - applications: list of application names + - application_categories: list of application category tags + - protocols: list of protocol names + - protocol_categories: list of protocol category tags + - config_name: config name of the group being edited, excluded from the name uniqueness check + + Returns: + a tuple with the sanitized name and the sanitized members, indexed by UCI list name + + Raises: + - ValidationError: if the name or any member is invalid, or if the group has no members + """ + if not isinstance(name, str) or not name.strip(): + raise ValidationError('name', 'name_required', name) + name = name.strip() + if len(name) > APPGROUP_NAME_MAX_LENGTH: + raise ValidationError('name', 'name_too_long', name) + for section, group in (utils.get_all_by_type(e_uci, 'dpi', 'appgroup') or {}).items(): + if section != config_name and group.get('ns_name', '').strip().lower() == name.lower(): + raise ValidationError('name', 'name_already_exists', name) + + members = { + 'app': __validate_appgroup_members('app', applications), + 'app_category': __validate_appgroup_members('app_category', application_categories), + 'proto': __validate_appgroup_members('proto', protocols), + 'proto_category': __validate_appgroup_members('proto_category', protocol_categories) + } + if not any(members.values()): + # an empty group would expand to an empty criteria, which takes the agent down + raise ValidationError('members', 'appgroup_is_empty', '') + + return name, members + + +def __save_appgroup_data(e_uci: EUci, config_name: str, name: str, members: dict[str, list[str]]): + e_uci.set('dpi', config_name, 'ns_name', name) + for uci_list, values in members.items(): + if values: + e_uci.set('dpi', config_name, uci_list, values) + else: + # an emptied list must be removed, not stored empty + e_uci.delete('dpi', config_name, uci_list) + + +def is_used_appgroup(e_uci: EUci, config_name: str) -> tuple[bool, list[str]]: + """ + Check if an application group is referenced by a rule. + + Args: + - e_uci: euci instance + - config_name: config name of the group + + Returns: + A tuple with: + - True if the group is referenced by at least one rule, False otherwise + - a list of the rule sections referencing it + """ + matches = [] + for section, rule in (utils.get_all_by_type(e_uci, 'dpi', 'rule') or {}).items(): + if config_name in rule.get('appgroup', []): + matches.append(f'dpi/{section}') + return len(matches) > 0, matches + + +def list_appgroups(e_uci: EUci, search: str = None, limit: int = None, page: int = 1, + used_info: bool = True) -> dict: + """ + List the application groups, ordered by name. + + Args: + - e_uci: euci instance + - search: search string, matched against the group name + - limit: limit the number of results, all of them if not given + - page: page number + - used_info: include the used and matches info + + Returns: + dict with the "data" list and the "meta" pagination info + """ + groups = [] + for section, group in (utils.get_all_by_type(e_uci, 'dpi', 'appgroup') or {}).items(): + data_group = {'id': section, 'name': group.get('ns_name', '')} + for uci_list, parameter in APPGROUP_MEMBERS.items(): + data_group[parameter] = list(group.get(uci_list, [])) + groups.append(data_group) + + # pagination needs a deterministic order, the UCI one is not + groups.sort(key=lambda group: group['name'].lower()) + + if search: + search = search.lower() + groups = [group for group in groups if search in group['name'].lower()] + + total = len(groups) if limit is not None: - result = result[limit * (page - 1):limit * page] + groups = groups[limit * (page - 1):limit * page] last_page = math.ceil(total / limit) else: last_page = 1 + if used_info: + # only for the returned page: every check walks the rules + for group in groups: + group['used'], group['matches'] = is_used_appgroup(e_uci, group['id']) + return { - 'data': result, + 'data': groups, 'meta': { 'last_page': last_page, 'total': total, @@ -262,235 +527,702 @@ def list_popular(e_uci: EUci, limit: int = None, page: int = 1) -> dict: } -def list_rules(e_uci: EUci) -> list[dict[str]]: +def add_appgroup(e_uci: EUci, name: str, applications: list[str] = None, + application_categories: list[str] = None, protocols: list[str] = None, + protocol_categories: list[str] = None) -> str: + """ + Store a new application group. + + Args: + - e_uci: euci instance + - name: name of the group + - applications: list of application names, as the engine reports them + - application_categories: list of application category tags + - protocols: list of protocol names, as the engine reports them + - protocol_categories: list of protocol category tags + + Returns: + config name of the group created + + Raises: + - ValidationError: if the name or any member is invalid, or if the group has no members + """ + name, members = __validate_appgroup(e_uci, name, applications, application_categories, protocols, + protocol_categories) + config_name = utils.get_random_id() + e_uci.set('dpi', config_name, 'appgroup') + __save_appgroup_data(e_uci, config_name, name, members) + e_uci.save('dpi') + return config_name + + +def edit_appgroup(e_uci: EUci, config_name: str, name: str, applications: list[str] = None, + application_categories: list[str] = None, protocols: list[str] = None, + protocol_categories: list[str] = None) -> str: + """ + Edit an application group. + + Args: + - e_uci: euci instance + - config_name: config name of the group to edit + - name: name of the group + - applications: list of application names, as the engine reports them + - application_categories: list of application category tags + - protocols: list of protocol names, as the engine reports them + - protocol_categories: list of protocol category tags + + Returns: + config name of the group edited + + Raises: + - ValidationError: if the group does not exist, or if the name or any member is invalid + """ + if e_uci.get('dpi', config_name, default=None) != 'appgroup': + raise ValidationError('id', 'appgroup_does_not_exists', config_name) + + name, members = __validate_appgroup(e_uci, name, applications, application_categories, protocols, + protocol_categories, config_name) + __save_appgroup_data(e_uci, config_name, name, members) + e_uci.save('dpi') + return config_name + + +def delete_appgroup(e_uci: EUci, config_name: str) -> str: """ - Index all rules + Delete an application group. Args: - e_uci: euci instance + - config_name: config name of the group to delete Returns: - list of dicts, each dict contains the property "config-name", "description", "enabled", "interface" and "blocks" - """ - rules = list[dict[str]]() - fetch_rules = utils.get_all_by_type(e_uci, 'dpi', 'rule') - - if not fetch_rules: - return rules - - for rule_name in fetch_rules.keys(): - # skipping rules with criteria, must be custom entries - if e_uci.get('dpi', rule_name, 'criteria', default=None) is None: - # load blocklist of applications and protocols - blocklist = __load_blocklist() - # get content of rule - rule = fetch_rules[rule_name] - # prepare the data to append to rules - data_rule = dict[str]() - data_rule['config-name'] = rule_name - data_rule['enabled'] = rule.get('enabled', '1') == '1' - data_rule['device'] = rule.get('device', '*') - # from device, get the interface - interface = utils.get_interface_from_device(e_uci, data_rule['device']) - if interface is not None: - data_rule['interface'] = interface - data_rule['action'] = rule.get('action') - # get the blocked applications/protocols - data_rule['criteria'] = list[dict[str]]() - - # filter by application - application_blocklist = [item for item in blocklist if item['type'] == 'application'] - for application in rule.get('application', []): - found_app = [item for item in application_blocklist if item['name'] == application] - # there's a possibility of not finding the application due to manual edit of the config - if len(found_app) > 0: - data_rule['criteria'].append(found_app[0]) - - # filter by protocol - protocol_blocklist = [item for item in blocklist if item['type'] == 'protocol'] - for protocol in rule.get('protocol', []): - found_protocol = [item for item in protocol_blocklist if item['name'] == protocol] - # there's a possibility of not finding the protocol due to manual edit of the config - if len(found_protocol) > 0: - data_rule['criteria'].append(found_protocol[0]) - - # append rule - rules.append(data_rule) + config name of the group deleted + + Raises: + - ValidationError: if the group does not exist or is referenced by a rule + """ + if e_uci.get('dpi', config_name, default=None) != 'appgroup': + raise ValidationError('id', 'appgroup_does_not_exists', config_name) + + used, matches = is_used_appgroup(e_uci, config_name) + if used: + raise ValidationError('id', 'appgroup_is_used', matches) + + e_uci.delete('dpi', config_name) + e_uci.save('dpi') + return config_name + + +def expand_appgroups(e_uci: EUci, config_names: list[str]) -> str: + """ + Expand application groups into the parenthesised OR block of an action criteria. + + Members of several groups are merged and deduplicated; categories are never expanded into their + applications, so a category keeps following the signature updates. Groups that do not exist are + skipped. + + Args: + - e_uci: euci instance + - config_names: config names of the groups to expand + + Returns: + the expression, e.g. `(app == 'netify.amazon' || proto == 'http/connect')`, or an empty string + if the groups hold no member + """ + terms = [] + seen = set[str]() + for uci_list in APPGROUP_MEMBERS: + for config_name in config_names: + if e_uci.get('dpi', config_name, default=None) != 'appgroup': + continue + for value in e_uci.get('dpi', config_name, uci_list, list=True, default=[]): + # protocol and category names are matched lowercased, application tags already are + value = value if uci_list == 'app' else value.lower() + term = f"{uci_list} == '{value}'" + if term not in seen: + seen.add(term) + terms.append(term) + + if not terms: + return '' + return f"({' || '.join(terms)})" + + +# Rules: ordered by priority, first match wins. A rule matches application groups only, optionally +# narrowed to a source; what it does is decided by its action. + +# 'allow' attaches a label no nft rule matches: it exists so a rule can win the first-match race and +# shield its traffic from the block rules below it +DPI_RULE_ACTIONS = ('block', 'allow') + +DPI_RULE_NAME_MAX_LENGTH = 64 + +DPI_RULE_POSITIONS = ('top', 'bottom') + + +def __sorted_rules(e_uci: EUci) -> list[tuple[str, dict]]: + """ + Rule sections in priority order. Rules with no priority, i.e. written by hand, go last keeping + their config order. + """ + def sort_key(item: tuple[str, dict]): + try: + return 0, int(item[1].get('priority')) + except (TypeError, ValueError): + return 1, 0 + + return sorted((utils.get_all_by_type(e_uci, 'dpi', 'rule') or {}).items(), key=sort_key) + + +def __renumber_rules(e_uci: EUci, order: list[str] = None): + """ + Give every rule a dense priority, starting at 1. The generator keeps 0 for its own action, so no + user rule can be evaluated before it. + """ + if order is None: + order = [section for section, _ in __sorted_rules(e_uci)] + for priority, section in enumerate(order, start=1): + e_uci.set('dpi', section, 'priority', priority) + + +def __is_rule(e_uci: EUci, config_name: str) -> bool: + return e_uci.get('dpi', config_name, default=None) == 'rule' + + +def __validate_rule_name(name: str) -> str: + if not isinstance(name, str) or not name.strip(): + raise ValidationError('name', 'name_required', name) + name = name.strip() + if len(name) > DPI_RULE_NAME_MAX_LENGTH: + raise ValidationError('name', 'name_too_long', name) + return name + + +def __validate_source(values: list[str]) -> list[str]: + """ + Validate the source of a rule: addresses, networks and ranges, IPv4 and IPv6 alike. An empty + source means the rule matches every host. + """ + sources = [] + for value in values or []: + if not isinstance(value, str) or not value.strip(): + raise ValidationError('source', 'invalid_source', value) + source = value.strip() + try: + if '-' in source: + first, last = (ipaddress.ip_address(part.strip()) for part in source.split('-', 1)) + if first.version != last.version or first > last: + raise ValueError(source) + else: + ipaddress.ip_network(source, strict=False) + except ValueError: + raise ValidationError('source', 'invalid_source', value) + if source not in sources: + sources.append(source) + return sources + + +def __validate_appgroups(e_uci: EUci, config_names: list[str], require_appgroup: bool = True) -> list[str]: + """ + Validate the groups a rule matches. At least one is required by default: a rule with a source and + no group would be an IP-level rule, which the firewall does better, and a rule with neither would + expand to an empty criteria. + + `require_appgroup=False` lifts that requirement for a rule migrated from a global exemption: it is + a source-only Allow rule by design, a shape the drawer cannot produce and never will. + """ + appgroups = [] + for config_name in config_names or []: + if not isinstance(config_name, str) or e_uci.get('dpi', config_name, default=None) != 'appgroup': + raise ValidationError('appgroups', 'appgroup_does_not_exists', config_name) + if config_name not in appgroups: + appgroups.append(config_name) + if require_appgroup and not appgroups: + raise ValidationError('appgroups', 'appgroups_required', appgroups) + return appgroups + + +def expand_source(values: list[str]) -> list[str]: + """ + Expand the source of a rule into values a criteria can match: ranges become CIDR blocks, addresses + and networks are left as they are. + + Args: + - values: source entries of the rule + + Returns: + list of addresses and CIDR blocks + """ + expanded = [] + for value in values: + if '-' in value: + first, last = (ipaddress.ip_address(part.strip()) for part in value.split('-', 1)) + expanded.extend(str(network) for network in ipaddress.summarize_address_range(first, last)) + else: + expanded.append(value) + return expanded + + +def build_rule_criteria(e_uci: EUci, rule: dict, require_appgroup: bool = True) -> str: + """ + Build the criteria of a managed rule: the source, when set, ANDed with the union of its groups. + + Args: + - e_uci: euci instance + - rule: the rule section, as returned by `uci.get_all` + - require_appgroup: when False, a rule with a source and no group emits a source-only criteria + instead of being treated as matching nothing. Used for a rule migrated from a global exemption, + the one shape of source-only Allow rule the drawer cannot produce and never will. + + Returns: + the semicolon-terminated expression, or an empty string when the rule matches nothing and must + not be emitted + """ + match = expand_appgroups(e_uci, list(rule.get('appgroup', []))) + sources = [f'local_ip == {source}' for source in expand_source(list(rule.get('source', [])))] + + if not match: + if require_appgroup or not sources: + return '' + return f"({' || '.join(sources)});" + + if sources: + return f"({' || '.join(sources)}) && {match};" + return f'{match};' + + +def list_rules(e_uci: EUci) -> list[dict]: + """ + List the rules in priority order, the order they are evaluated in. + + Rules hidden with `ns_visible '0'` are left out: they are system entries the user must not touch. + + Args: + - e_uci: euci instance + + Returns: + list of dicts, each dict contains the property "id", "name", "enabled", "action", "source", + "appgroups", "managed" and "index", plus "criteria" for the rules the API did not create + """ + groups = {section: group.get('ns_name', '') + for section, group in (utils.get_all_by_type(e_uci, 'dpi', 'appgroup') or {}).items()} + rules = [] + for section, rule in __sorted_rules(e_uci): + if rule.get('ns_visible', '1') == '0': + continue + data_rule = { + 'id': section, + 'name': rule.get('ns_name', ''), + 'enabled': rule.get('enabled', '1') == '1', + 'action': rule.get('action', ''), + 'source': list(rule.get('source', [])), + 'appgroups': [{'id': group, 'name': groups.get(group, group)} + for group in rule.get('appgroup', [])], + # a rule the UI did not create can be renamed, toggled, reordered and deleted, but not edited + 'managed': rule.get('ns_managed', '0') == '1', + 'index': len(rules) + } + if 'criteria' in rule: + data_rule['criteria'] = rule.get('criteria') + rules.append(data_rule) return rules -def __save_rule_data(e_uci: EUci, config_name: str, enabled: bool, device: str, action: str, applications: list[str], - protocols: list[str]): +def __save_rule_data(e_uci: EUci, config_name: str, name: str, enabled: bool, action: str, + source: list[str], appgroups: list[str]): + e_uci.set('dpi', config_name, 'ns_name', name) + e_uci.set('dpi', config_name, 'ns_managed', '1') e_uci.set('dpi', config_name, 'enabled', enabled) - e_uci.set('dpi', config_name, 'device', device) e_uci.set('dpi', config_name, 'action', action) - e_uci.set('dpi', config_name, 'application', applications) - e_uci.set('dpi', config_name, 'protocol', protocols) + if appgroups: + e_uci.set('dpi', config_name, 'appgroup', appgroups) + else: + # a rule migrated from a global exemption has no group: the option must go, not stay empty + e_uci.delete('dpi', config_name, 'appgroup') + if source: + e_uci.set('dpi', config_name, 'source', source) + else: + # no source means every host: the option must go, not stay empty + e_uci.delete('dpi', config_name, 'source') -def __save_exemption_data(e_uci: EUci, config_name: str, criteria: str, description: str, enabled: bool): - e_uci.set('dpi', config_name, 'enabled', enabled) - e_uci.set('dpi', config_name, 'criteria', criteria) - e_uci.set('dpi', config_name, 'description', description) -def __toggle_engine(e_uci: EUci): - count_enabled = 0 - for section in e_uci.get_all('dpi'): - if e_uci.get('dpi', section, default="") == "rule" and e_uci.get('dpi', section, 'enabled', default="0") == "1": - count_enabled = count_enabled + 1 +def add_rule(e_uci: EUci, name: str, enabled: bool, action: str, source: list[str], + appgroups: list[str], position: str = 'bottom', require_appgroup: bool = True) -> str: + """ + Store a new rule. - if count_enabled > 0: - e_uci.set('dpi', 'config', 'enabled', '1') - else: - e_uci.set('dpi', 'config', 'enabled', '0') + Args: + - e_uci: euci instance + - name: name of the rule + - enabled: enable the rule + - action: 'block' or 'allow' + - source: list of addresses, networks or ranges, empty to match every host + - appgroups: config names of the application groups the rule matches, at least one unless + `require_appgroup` is False + - position: 'top' to evaluate the rule before every other one, 'bottom' after them + - require_appgroup: set to False only when migrating a global exemption into a source-only Allow + rule, a shape the drawer cannot produce and never will -def add_rule(e_uci: EUci, enabled: bool, device: str, action: str, applications: list[str], - protocols: list[str]) -> str: + Returns: + config name of the rule created + + Raises: + - ValidationError: if any argument is invalid """ - Store a new rule + name = __validate_rule_name(name) + if action not in DPI_RULE_ACTIONS: + raise ValidationError('action', 'invalid_action', action) + if position not in DPI_RULE_POSITIONS: + raise ValidationError('position', 'invalid_position', position) + source = __validate_source(source) + appgroups = __validate_appgroups(e_uci, appgroups, require_appgroup) + + config_name = utils.get_random_id() + e_uci.set('dpi', config_name, 'rule') + __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups) + + order = [section for section, _ in __sorted_rules(e_uci) if section != config_name] + order.insert(0, config_name) if position == 'top' else order.append(config_name) + __renumber_rules(e_uci, order) + + __toggle_engine(e_uci) + e_uci.save('dpi') + return config_name + + +def edit_rule(e_uci: EUci, config_name: str, name: str, enabled: bool, action: str, source: list[str], + appgroups: list[str]) -> str: + """ + Edit a rule. Only rules created through the API can be edited: a rule carrying a hand-written + criteria has no source and no group to fill the form with, and rewriting it would change what it + matches. Args: - e_uci: euci instance - - description: description of the rule + - config_name: config name of the rule to edit + - name: name of the rule - enabled: enable the rule - - action: apply specific action to rule, can be 'block', 'bulk', 'best_effort', 'video' or 'voice' - - device: device to listen and apply the rule on - - applications: list of applications to block - - protocols: list of protocols to block + - action: 'block' or 'allow' + - source: list of addresses, networks or ranges, empty to match every host + - appgroups: config names of the application groups the rule matches, at least one Returns: - config name of the rule created + config name of the rule edited + + Raises: + - ValidationError: if the rule does not exist, is not managed, or any argument is invalid """ - rule_name = utils.get_random_id() - e_uci.set('dpi', rule_name, 'rule') - __save_rule_data(e_uci, rule_name, enabled, device, action, applications, protocols) + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) + if e_uci.get('dpi', config_name, 'ns_managed', default='0') != '1': + raise ValidationError('id', 'rule_not_managed', config_name) + + name = __validate_rule_name(name) + if action not in DPI_RULE_ACTIONS: + raise ValidationError('action', 'invalid_action', action) + source = __validate_source(source) + appgroups = __validate_appgroups(e_uci, appgroups) + + __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups) __toggle_engine(e_uci) e_uci.save('dpi') - return rule_name + return config_name -def delete_rule(e_uci: EUci, config_name: str): +def delete_rule(e_uci: EUci, config_name: str) -> str: """ - Delete a rule + Delete a rule and close the gap its priority leaves behind. Args: - e_uci: euci instance - config_name: config name of the rule to delete + + Returns: + config name of the rule deleted + + Raises: + - ValidationError: if the rule does not exist """ + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) + e_uci.delete('dpi', config_name) + __renumber_rules(e_uci) __toggle_engine(e_uci) e_uci.save('dpi') + return config_name -def edit_rule(e_uci: EUci, config_name: str, enabled: bool, device: str, action: str, applications: list[str], - protocols: list[str]): +def rename_rule(e_uci: EUci, config_name: str, name: str) -> str: """ - Edit a rule + Rename a rule, managed or not. Args: - e_uci: euci instance - - config_name: rule to change - - enabled: enable the rule - - device: device to listen and apply the rule on - - action: apply specific action to rule, can be 'block', 'bulk', 'best_effort', 'video' or 'voice' - - applications: array of applications to block - - protocols: array of protocols to block + - config_name: config name of the rule to rename + - name: new name of the rule - Raises - - ValidationError: if the config name is invalid - """ - if e_uci.get('dpi', config_name, default=None) is None: - raise ValidationError('config-name', 'invalid', config_name) + Returns: + config name of the rule renamed - __save_rule_data(e_uci, config_name, enabled, device, action, applications, protocols) - __toggle_engine(e_uci) + Raises: + - ValidationError: if the rule does not exist or the name is invalid + """ + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) + e_uci.set('dpi', config_name, 'ns_name', __validate_rule_name(name)) e_uci.save('dpi') + return config_name -def list_exemptions(e_uci: EUci) -> list[dict[str]]: + +def enable_rule(e_uci: EUci, config_name: str) -> str: """ - Index all global exemptions + Enable a rule, managed or not. Args: - e_uci: euci instance + - config_name: config name of the rule to enable Returns: - list of dicts, each dict contains the property "config-name", "description", "enabled", "criteria" - """ - exemptions = list[dict[str]]() - fetch_ex = utils.get_all_by_type(e_uci, 'dpi', 'exemption') + config name of the rule enabled - if not fetch_ex: - return exemptions - for ex_name in fetch_ex.keys(): - # get content of exemption - ex = fetch_ex[ex_name] - # prepare the data to append to rules - data_ex = dict[str]() - data_ex['config-name'] = ex_name - data_ex['enabled'] = ex.get('enabled', '1') == '1' - data_ex['criteria'] = ex.get('criteria', '') - data_ex['description'] = ex.get('description', '') - # append exemption - exemptions.append(data_ex) + Raises: + - ValidationError: if the rule does not exist + """ + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) - return exemptions + e_uci.set('dpi', config_name, 'enabled', '1') + __toggle_engine(e_uci) + e_uci.save('dpi') + return config_name -def add_exemption(e_uci: EUci, criteria: str, description: str, enabled: bool): +def disable_rule(e_uci: EUci, config_name: str) -> str: """ - Store a new global exemption + Disable a rule, managed or not. Args: - e_uci: euci instance - - criteria: exemption criteria, usually it's an IP address - - description: description of the rule - - enabled: enable the exemption + - config_name: config name of the rule to disable Returns: - config name of the exemption created + config name of the rule disabled + + Raises: + - ValidationError: if the rule does not exist """ - ex_list = utils.get_all_by_type(e_uci, 'dpi', 'exemption') - for ex_name in ex_list: - ex = ex_list[ex_name] - if ex.get('criteria', '') == criteria: - raise ValidationError('criteria', 'criteria_already_exists', criteria) + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) - ex_name = utils.get_random_id() - e_uci.set('dpi', ex_name, 'exemption') - __save_exemption_data(e_uci, ex_name, criteria, description, enabled) + e_uci.set('dpi', config_name, 'enabled', '0') + __toggle_engine(e_uci) e_uci.save('dpi') - return ex_name + return config_name -def delete_exemption(e_uci: EUci, config_name: str): +def order_rules(e_uci: EUci, order: list[str]) -> list[str]: """ - Delete a global exemption + Reorder every rule, renumbering the priorities densely from the given order. + + The whole set must be listed, hidden rules included: a partial order would silently move the rules + left out, which with a paginated list is a rule the caller never saw. Args: - e_uci: euci instance - - config_name: config name of the rule to delete + - order: config names of every rule, in the order they must be evaluated + + Returns: + the list of the ordered rules + + Raises: + - ValidationError: if the order does not name every rule exactly once """ - e_uci.delete('dpi', config_name) + rules = [section for section, _ in __sorted_rules(e_uci)] + for section in order: + if section not in rules: + raise ValidationError('order', 'rule_not_found', section) + if len(order) != len(set(order)) or len(order) != len(rules): + raise ValidationError('order', 'invalid_order', order) + + __renumber_rules(e_uci, order) e_uci.save('dpi') + return order + + +# Migration: one-time, in-place bump of /etc/config/dpi from the schema used before application groups +# existed. A rule from that schema carries `device`, `application`, `protocol` or `category` — fields +# only that API ever wrote, so their presence is a permanent, unambiguous marker regardless of whether +# `priority` was since assigned to it by an unrelated add/edit/delete. A global exemption is identified +# by its section type, which this migration removes once every exemption has become a rule. + +def freeze_legacy_criteria(e_uci: EUci, rule: dict) -> str: + """ + Build the criteria of a rule written before application groups existed, the way `dpi-config` has + always generated it: source, application, protocol and category turned into an expression narrowed + to the rule's device, with the VLAN rewrite applied when the device names a VLAN. Used by the + generator for a legacy rule the migration has not converted yet, and by the migration itself to + freeze a rule's behaviour verbatim before clearing those fields. + + Args: + - e_uci: euci instance + - rule: the rule section, as returned by `uci.get_all` + + Returns: + the semicolon-terminated expression + """ + device = rule.get('device', '*') + + if 'criteria' in rule: + # criteria has precedence over source, protocol, category and application + criteria = rule['criteria'].replace('"', "'") + else: + sources = [] + for source in rule.get('source', []): + if objects.is_object_id(source): + for ip in objects.get_object_ips(e_uci, source): + sources.append(f'local_ip == {ip}') + else: + sources.append(f'local_ip == {source}') + + applications = [] + for app in rule.get('application', []): + applications.append(f"app == '{app}'") + for proto in rule.get('protocol', []): + applications.append(f"proto == '{proto.lower()}'") + for cat in rule.get('category', []): + applications.append(f"category == '{cat.lower()}'") + + sources_s = ' or '.join(sources) + applications_s = ' or '.join(applications) + criteria = f"(iface_nfq_src == '{device}' or iface_nfq_dst == '{device}') && " + if len(sources) < 1: + criteria += f'({applications_s}) ;' + elif len(applications) < 1: + criteria += f'({sources_s}) ;' + else: + criteria += f'({sources_s}) && ({applications_s}) ;' + + vlan_id = None + base_if = None + for item in utils.get_all_by_type(e_uci, 'network', 'device').values(): + if item.get('vid', None) is not None and item.get('name', '') == device: + vlan_id = item.get('vid', None) + base_if = item.get('ifname', None) + break + if vlan_id is not None and base_if is not None: + criteria = f'vlan_id == {vlan_id} && {criteria}' -def edit_exemption(e_uci: EUci, config_name: str, criteria: str, description: str, enabled: bool): + return criteria + + +def __exemption_source(e_uci: EUci, criteria: str) -> list[str] | None: + """ + Turn a global exemption's criteria into a rule source. A firewall object is expanded into its + member addresses; a plain address, network or range is kept as is. + + Returns: + the source list, or None when the criteria is empty or does not resolve to at least one + address, telling the caller to fall back to an unmanaged rule carrying it verbatim """ - Edit a global exemption + if not criteria: + return None + if objects.is_object_id(criteria): + return objects.get_object_ips(e_uci, criteria) or None + try: + return __validate_source([criteria]) + except ValidationError: + return None + + +def migrate_schema(e_uci: EUci) -> bool: + """ + Migrate every rule and exemption from the schema used before application groups existed. + + Each legacy rule (`device`, `application`, `protocol` or `category` set) is turned into an unmanaged + rule with its behaviour frozen verbatim into `criteria` — see `freeze_legacy_criteria`. A rule whose + action is a retired QoS value, carries a per-rule exemption (a field no recent API ever wrote), or + ends up matching nothing is dropped instead, with a log line naming it. + + Each global exemption becomes an Allow rule at the top of the list: a managed, source-only rule + (`require_appgroup=False`) when its criteria is a plain address, CIDR or firewall object; an + unmanaged one carrying the criteria verbatim otherwise. Disabled exemptions become disabled rules. + The `exemption` section type, `firewall_exemption` and `popular_filters` are then removed. + + Safe to call unconditionally on every boot: a box with nothing left in the old schema returns False + without touching UCI. Args: - e_uci: euci instance - - config_name: rule to change - - criteria: exemption criteria, usually it's an IP address - - description: description of the rule - - enabled: enable the exemption - Raises - - ValidationError: if the config name is invalid + Returns: + True if the config was changed (and therefore committed and reloaded), False otherwise """ - if e_uci.get('dpi', config_name, default=None) is None: - raise ValidationError('config-name', 'invalid', config_name) + exemptions = utils.get_all_by_type(e_uci, 'dpi', 'exemption') or {} + rules = utils.get_all_by_type(e_uci, 'dpi', 'rule') or {} + legacy_fields = ('device', 'application', 'protocol', 'category') + legacy_rules = {section for section, rule in rules.items() + if any(field in rule for field in legacy_fields)} + + if not exemptions and not legacy_rules: + return False + + order = [] + + exemption_count = 0 + for section, exemption in exemptions.items(): + exemption_count += 1 + name = f'Migrated exception {exemption_count}' + enabled = exemption.get('enabled', '1') == '1' + criteria = (exemption.get('criteria') or '').strip() + source = __exemption_source(e_uci, criteria) + + e_uci.delete('dpi', section) + if source is not None: + rule_id = add_rule(e_uci, name, enabled, 'allow', source, [], require_appgroup=False) + else: + rule_id = utils.get_random_id() + e_uci.set('dpi', rule_id, 'rule') + e_uci.set('dpi', rule_id, 'ns_name', name) + e_uci.set('dpi', rule_id, 'enabled', '1' if enabled else '0') + e_uci.set('dpi', rule_id, 'action', 'allow') + e_uci.set('dpi', rule_id, 'criteria', criteria if criteria.endswith(';') else f'{criteria};') + order.append(rule_id) + + rule_count = 0 + for section, rule in rules.items(): + if section not in legacy_rules: + order.append(section) + continue - __save_exemption_data(e_uci, config_name, criteria, description, enabled) + action = rule.get('action') + if action != 'block': + syslog.syslog(syslog.LOG_WARNING, f"dpi migration: dropping {section}, unsupported action {action!r}") + e_uci.delete('dpi', section) + continue + if rule.get('exemption'): + syslog.syslog(syslog.LOG_WARNING, + f"dpi migration: dropping the per-rule exemption on {section}, no longer supported") + if not any(rule.get(field) for field in ('criteria', 'source', 'application', 'protocol', 'category')): + syslog.syslog(syslog.LOG_WARNING, f"dpi migration: dropping {section}, it matches nothing") + e_uci.delete('dpi', section) + continue + + rule_count += 1 + criteria = freeze_legacy_criteria(e_uci, rule) + for field in (*legacy_fields, 'source', 'exemption'): + e_uci.delete('dpi', section, field) + e_uci.set('dpi', section, 'criteria', criteria) + e_uci.set('dpi', section, 'ns_name', f'Migrated rule {rule_count}') + order.append(section) + + __renumber_rules(e_uci, order) + e_uci.delete('dpi', 'config', 'firewall_exemption') + e_uci.delete('dpi', 'config', 'popular_filters') + __toggle_engine(e_uci) e_uci.save('dpi') + __apply(e_uci) + return True diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py index 2e049dba2..2bcdb8072 100644 --- a/packages/python3-nethsec/tests/test_dpi.py +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -4,7 +4,7 @@ from euci import EUci from pytest_mock import MockFixture -from nethsec import dpi +from nethsec import dpi, utils from nethsec.utils import ValidationError applications_file = """ @@ -197,6 +197,18 @@ } } +application_output = """ + 133: netify.netflix + 10119: netify.linkedin + 10552: netify.tesla + 10195: netify.avira + 10194: netify.sophos + 10244: netify.bbc + 10362: netify.hulu + 10118: netify.lets-encrypt + 199: netify.snapchat +""" + protocol_output = """ 116: Warcraft3 117: LotusNotes @@ -257,11 +269,6 @@ list protocol 'HTTP/Connect' option device 'eth1' option enabled 1 - -config exemption exemp1 - option criteria '192.168.1.1' - option description 'my host' - option enabled 1 """ netifyd_config = """ @@ -349,6 +356,15 @@ """ +@pytest.fixture(autouse=True) +def mock_apply(mocker: MockFixture): + """ + add/delete of rules and appgroups commit the dpi config and reload the dpi service right away; + tests only care about the resulting uci state, not about actually reloading a system service. + """ + return mocker.patch('nethsec.dpi.__apply') + + @pytest.fixture def e_uci(tmp_path: pathlib.Path) -> EUci: conf_dir = tmp_path.joinpath('conf') @@ -376,6 +392,7 @@ def e_uci_with_data(e_uci: EUci): @pytest.fixture def mock_load(mocker): mocker.patch('nethsec.dpi.__load_applications', return_value=applications) + mocker.patch('nethsec.dpi.load_applications', return_value=applications) mocker.patch('nethsec.dpi.__load_application_categories', return_value=application_categories) mocker.patch('nethsec.dpi.load_protocols', return_value=protocols) mocker.patch('nethsec.dpi.__load_protocol_categories', return_value=protocol_categories) @@ -398,6 +415,13 @@ def test_load_protocol_categories(mocker: MockFixture, data: str): assert dpi.__load_protocol_categories() == protocol_categories +def test_load_applications_from_engine(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = bytes(application_output, 'utf-8') + mocker.patch('subprocess.run', return_value=process_result) + assert dpi.load_applications() == applications + + def test_load_protocols(mocker: MockFixture): process_result = mocker.stub('subprocess_return') process_result.stdout = bytes(protocol_output, 'utf-8') @@ -675,393 +699,805 @@ def test_index_applications_paginate(mock_load): } -def test_list_empty_rules(e_uci, mock_load): - assert dpi.list_rules(e_uci) == [] +# application groups +category_output = """ + 1: application: adult + 5: application: cybersecurity + 12: application: games + 5: protocol: games + 18: protocol: web +""" -def test_list_rules(e_uci_with_data, mock_load): - assert dpi.list_rules(e_uci_with_data) == [ - { - 'config-name': 'rule0', - 'enabled': True, - 'interface': 'GREEN_1', - 'device': 'eth0', - 'action': 'block', - 'criteria': [ - { - 'id': 10119, - 'name': 'netify.linkedin', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 199, - 'name': 'netify.snapchat', - 'type': 'application', - 'category': { - 'name': 'last' - } - }, - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - }, - { - 'config-name': 'rule1', - 'enabled': False, - 'interface': 'GREEN_2', - 'device': 'eth4', - 'action': 'block', - 'criteria': [ - { - 'id': 10552, - 'name': 'netify.tesla', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - } - ] - }, - { - 'config-name': 'rule3', - 'enabled': True, - 'interface': 'RED_1', - 'device': 'eth1', - 'action': 'video', - 'criteria': [ - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - } - ] +categories = { + 'application': {1: 'adult', 5: 'cybersecurity', 12: 'games'}, + 'protocol': {5: 'games', 18: 'web'} +} -def test_store_rule(e_uci, mock_load): - rule_created = dpi.add_rule(e_uci, True, 'eth0', 'best_effort', - ['netify.linkedin', 'netify.avira', 'netify.netflix'], ['LotusNotes', 'SFlow']) - assert dpi.list_rules(e_uci) == [ - { - 'config-name': rule_created, - 'enabled': True, - 'interface': 'GREEN_1', - 'device': 'eth0', - 'action': 'best_effort', - 'criteria': [ - { - 'id': 10119, - 'name': 'netify.linkedin', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 10195, - 'name': 'netify.avira', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 133, - 'name': 'netify.netflix', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 117, - 'name': 'LotusNotes', - 'type': 'protocol', - }, - { - 'id': 129, - 'name': 'SFlow', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - } - ] - assert(e_uci.get("dpi", "config", "enabled") == "1") +@pytest.fixture +def mock_vocabularies(mocker): + """Every member kind resolves against the engine, the catalogs are never reached.""" + mocker.patch('nethsec.dpi.load_applications', return_value=applications) + mocker.patch('nethsec.dpi.load_protocols', return_value=protocols) + mocker.patch('nethsec.dpi.load_categories', return_value=categories) -def test_delete_rule(e_uci_with_data, mock_load): - dpi.delete_rule(e_uci_with_data, 'rule1') - dpi.delete_rule(e_uci_with_data, 'rule0') - assert dpi.list_rules(e_uci_with_data) == [ - { - 'config-name': 'rule3', - 'enabled': True, - 'interface': 'RED_1', - 'device': 'eth1', - 'action': 'video', - 'criteria': [ - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - } - ] - dpi.delete_rule(e_uci_with_data, 'rule2') - dpi.delete_rule(e_uci_with_data, 'rule3') - assert(e_uci_with_data.get("dpi", "config", "enabled") == "0") +@pytest.fixture +def e_uci_appgroups(e_uci_with_data, mock_vocabularies): + return e_uci_with_data -def test_edit_rule(e_uci_with_data, mock_load): - dpi.edit_rule(e_uci_with_data, 'rule0', False, 'eth0', 'voice', [], - ['HTTP/Connect', 'LotusNotes']) - assert dpi.list_rules(e_uci_with_data) == [ - { - 'config-name': 'rule0', - 'enabled': False, - 'interface': 'GREEN_1', - 'device': 'eth0', - 'action': 'voice', - 'criteria': [ - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - }, - { - 'id': 117, - 'name': 'LotusNotes', - 'type': 'protocol', - } - ] - }, - { - 'config-name': 'rule1', - 'enabled': False, - 'interface': 'GREEN_2', - 'device': 'eth4', - 'action': 'block', - 'criteria': [ - { - 'id': 10552, - 'name': 'netify.tesla', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - } - ] - }, - { - 'config-name': 'rule3', - 'enabled': True, - 'interface': 'RED_1', - 'device': 'eth1', - 'action': 'video', - 'criteria': [ - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - } - ] +def test_load_categories(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = bytes(category_output, 'utf-8') + mocker.patch('subprocess.run', return_value=process_result) + assert dpi.load_categories() == categories + + +def test_add_appgroup(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], ['cybersecurity'], + ['Dropbox'], ['web']) + assert e_uci_appgroups.get('dpi', group_id) == 'appgroup' + assert e_uci_appgroups.get('dpi', group_id, 'ns_name') == 'Group one' + assert e_uci_appgroups.get('dpi', group_id, 'app', list=True) == ('netify.netflix',) + assert e_uci_appgroups.get('dpi', group_id, 'app_category', list=True) == ('cybersecurity',) + assert e_uci_appgroups.get('dpi', group_id, 'proto', list=True) == ('Dropbox',) + assert e_uci_appgroups.get('dpi', group_id, 'proto_category', list=True) == ('web',) + + +def test_add_appgroup_strips_and_deduplicates(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, ' Group one ', + [' netify.netflix ', 'netify.netflix', 'NETIFY.NETFLIX'], [], [], []) + assert e_uci_appgroups.get('dpi', group_id, 'ns_name') == 'Group one' + assert e_uci_appgroups.get('dpi', group_id, 'app', list=True) == ('netify.netflix',) + + +def test_add_appgroup_with_one_kind_only(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Only protocols', protocols=['HTTP/Connect']) + assert e_uci_appgroups.get('dpi', group_id, 'proto', list=True) == ('HTTP/Connect',) + assert e_uci_appgroups.get('dpi', group_id, 'app', default=None) is None + + +def test_add_appgroup_requires_a_name(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, ' ', ['netify.netflix']) + assert err.value.args[0] == 'name' + assert err.value.args[1] == 'name_required' + + +def test_add_appgroup_refuses_a_long_name(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'x' * 65, ['netify.netflix']) + assert err.value.args[1] == 'name_too_long' + + +def test_add_appgroup_refuses_a_duplicated_name(e_uci_appgroups): + dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'group ONE', ['netify.tesla']) + assert err.value.args[1] == 'name_already_exists' + + +def test_add_appgroup_refuses_an_empty_group(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Empty', [], [], [], []) + assert err.value.args[0] == 'members' + assert err.value.args[1] == 'appgroup_is_empty' + + +@pytest.mark.parametrize('value', ["netify.netflix'", 'netify.netflix;', 'netify.(netflix)', + 'netify.netflix\\', 'netify\nnetify.netflix', ' ']) +def test_add_appgroup_refuses_expression_breaking_members(e_uci_appgroups, value): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', [value]) + assert err.value.args[0] == 'applications' + assert err.value.args[1] == 'invalid_application' + + +def test_add_appgroup_refuses_unknown_members(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.nonexistent']) + assert err.value.args[1] == 'invalid_application' + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', protocols=['nonexistent']) + assert err.value.args[1] == 'invalid_protocol' + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', application_categories=['web']) + assert err.value.args[1] == 'invalid_application_category' + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', protocol_categories=['cybersecurity']) + assert err.value.args[1] == 'invalid_protocol_category' + + +def test_add_appgroup_refuses_a_protocol_named_by_its_catalog_tag(e_uci_appgroups): + # the catalog calls it "http-connect", but only the name the engine reports can be matched + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', protocols=['http-connect']) + assert err.value.args[1] == 'invalid_protocol' + + +def test_add_appgroup_matches_members_case_insensitively(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', protocols=['http/connect']) + assert e_uci_appgroups.get('dpi', group_id, 'proto', list=True) == ('http/connect',) + + +def test_add_appgroup_skips_validation_without_a_vocabulary(e_uci_with_data, mocker): + mocker.patch('nethsec.dpi.load_applications', side_effect=FileNotFoundError) + mocker.patch('nethsec.dpi.__load_catalog_tags', side_effect=FileNotFoundError) + group_id = dpi.add_appgroup(e_uci_with_data, 'Group one', ['netify.unknown-to-everything']) + assert e_uci_with_data.get('dpi', group_id, 'app', list=True) == ('netify.unknown-to-everything',) + + +def test_edit_appgroup(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], ['cybersecurity']) + dpi.edit_appgroup(e_uci_appgroups, group_id, 'Group two', ['netify.tesla']) + assert e_uci_appgroups.get('dpi', group_id, 'ns_name') == 'Group two' + assert e_uci_appgroups.get('dpi', group_id, 'app', list=True) == ('netify.tesla',) + # an emptied list is removed, not stored empty + assert e_uci_appgroups.get('dpi', group_id, 'app_category', default=None) is None + + +def test_edit_appgroup_keeps_its_own_name(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + dpi.edit_appgroup(e_uci_appgroups, group_id, 'Group one', ['netify.tesla']) + assert e_uci_appgroups.get('dpi', group_id, 'app', list=True) == ('netify.tesla',) + + +def test_edit_appgroup_of_unknown_id(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.edit_appgroup(e_uci_appgroups, 'ns_nonexistent', 'Group one', ['netify.netflix']) + assert err.value.args[1] == 'appgroup_does_not_exists' + + +def test_edit_appgroup_refuses_a_rule_id(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.edit_appgroup(e_uci_appgroups, 'rule0', 'Group one', ['netify.netflix']) + assert err.value.args[1] == 'appgroup_does_not_exists' + + +def test_delete_appgroup(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + dpi.delete_appgroup(e_uci_appgroups, group_id) + assert e_uci_appgroups.get('dpi', group_id, default=None) is None + +def test_add_appgroup_leaves_the_change_pending(e_uci_appgroups, mock_apply): + dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + mock_apply.assert_not_called() -def test_edit_rule_with_missing_rule(e_uci): + +def test_delete_appgroup_leaves_the_change_pending(e_uci_appgroups, mock_apply): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + mock_apply.reset_mock() + dpi.delete_appgroup(e_uci_appgroups, group_id) + mock_apply.assert_not_called() + + +def test_delete_appgroup_of_unknown_id(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.delete_appgroup(e_uci_appgroups, 'ns_nonexistent') + assert err.value.args[1] == 'appgroup_does_not_exists' + + +def test_delete_appgroup_in_use(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + e_uci_appgroups.set('dpi', 'rule0', 'appgroup', [group_id]) with pytest.raises(ValidationError) as err: - dpi.edit_rule(e_uci, 'rule0', False, 'eth0', 'block', [], []) - - assert err.value.args[0] == 'config-name' - assert err.value.args[1] == 'invalid' - assert err.value.args[2] == 'rule0' - - -def test_list_interfaces(e_uci_with_data): - assert dpi.list_devices(e_uci_with_data).index({ - 'interface': 'GREEN_1', - 'device': 'eth0' - }) != -1 - assert dpi.list_devices(e_uci_with_data).index({ - 'interface': 'GREEN_2', - 'device': 'eth4' - }) != -1 - with pytest.raises(ValueError): - assert dpi.list_devices(e_uci_with_data).index({ - 'interface': 'RED_1', - 'device': 'eth1' - }) - - -def test_list_popular(e_uci_with_data, mock_load): - assert dpi.list_popular(e_uci_with_data) == { + dpi.delete_appgroup(e_uci_appgroups, group_id) + assert err.value.args[1] == 'appgroup_is_used' + assert err.value.args[2] == ['dpi/rule0'] + + +def test_is_used_appgroup(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + assert dpi.is_used_appgroup(e_uci_appgroups, group_id) == (False, []) + e_uci_appgroups.set('dpi', 'rule0', 'appgroup', [group_id]) + e_uci_appgroups.set('dpi', 'rule1', 'appgroup', [group_id]) + used, matches = dpi.is_used_appgroup(e_uci_appgroups, group_id) + assert used + assert sorted(matches) == ['dpi/rule0', 'dpi/rule1'] + + +def test_list_appgroups(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], ['cybersecurity'], + ['Dropbox'], ['web']) + assert dpi.list_appgroups(e_uci_appgroups) == { 'data': [ { - 'id': 133, - 'name': 'netify.netflix', - 'type': 'application', - 'category': { - 'name': 'unknown' - }, - 'missing': False - }, - { - 'id': 10362, - 'name': 'netify.hulu', - 'type': 'application', - 'category': { - 'name': 'first-category' - }, - 'missing': False - }, - { - 'name': 'netify.whatsapp', - 'missing': True - }, - { - 'name': 'netify.facebook', - 'missing': True - }, - { - 'id': 10194, - 'name': 'netify.sophos', - 'type': 'application', - 'category': { - 'name': 'first-category' - }, - 'missing': False - }, - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - }, - 'missing': False - }, - { - 'id': 121, - 'name': 'Dropbox', - 'type': 'protocol', - 'category': { - 'name': 'base' - }, - 'missing': False + 'id': group_id, + 'name': 'Group one', + 'applications': ['netify.netflix'], + 'application_categories': ['cybersecurity'], + 'protocols': ['Dropbox'], + 'protocol_categories': ['web'], + 'used': False, + 'matches': [] } ], 'meta': { 'last_page': 1, - 'total': 7 + 'total': 1 } } -def test_list_popular_with_limits(e_uci_with_data, mock_load): - assert dpi.list_popular(e_uci_with_data, limit=2, page=2) == { - 'data': [ - { - 'name': 'netify.whatsapp', - 'missing': True - }, - { - 'name': 'netify.facebook', - 'missing': True - } - ], - 'meta': { - 'last_page': 4, - 'total': 7 - } - } - assert dpi.list_popular(e_uci_with_data, limit=2, page=3) == { - 'data': [ - { - 'id': 10194, - 'name': 'netify.sophos', - 'type': 'application', - 'category': { - 'name': 'first-category' - }, - 'missing': False - }, - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - }, - 'missing': False - } - ], - 'meta': { - 'last_page': 4, - 'total': 7 - } - } +def test_list_appgroups_when_empty(e_uci_appgroups): + assert dpi.list_appgroups(e_uci_appgroups) == {'data': [], 'meta': {'last_page': 1, 'total': 0}} + + +def test_list_appgroups_is_ordered_by_name(e_uci_appgroups): + dpi.add_appgroup(e_uci_appgroups, 'zeta', ['netify.netflix']) + dpi.add_appgroup(e_uci_appgroups, 'Alpha', ['netify.tesla']) + assert [group['name'] for group in dpi.list_appgroups(e_uci_appgroups)['data']] == ['Alpha', 'zeta'] + + +def test_list_appgroups_pagination(e_uci_appgroups): + for index in range(5): + dpi.add_appgroup(e_uci_appgroups, f'Group {index}', ['netify.netflix']) + result = dpi.list_appgroups(e_uci_appgroups, limit=2, page=2) + assert [group['name'] for group in result['data']] == ['Group 2', 'Group 3'] + assert result['meta'] == {'last_page': 3, 'total': 5} + + +def test_list_appgroups_search(e_uci_appgroups): + dpi.add_appgroup(e_uci_appgroups, 'Streaming', ['netify.netflix']) + dpi.add_appgroup(e_uci_appgroups, 'Business', ['netify.tesla']) + result = dpi.list_appgroups(e_uci_appgroups, search='STREAM') + assert [group['name'] for group in result['data']] == ['Streaming'] + assert result['meta'] == {'last_page': 1, 'total': 1} + + +def test_list_appgroups_reports_usage(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + e_uci_appgroups.set('dpi', 'rule0', 'appgroup', [group_id]) + group = dpi.list_appgroups(e_uci_appgroups)['data'][0] + assert group['used'] + assert group['matches'] == ['dpi/rule0'] + + +def test_expand_appgroups(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], ['cybersecurity'], + ['HTTP/Connect'], ['web']) + assert dpi.expand_appgroups(e_uci_appgroups, [group_id]) == ( + "(app == 'netify.netflix' || app_category == 'cybersecurity'" + " || proto == 'http/connect' || proto_category == 'web')" + ) + + +def test_expand_appgroups_merges_and_deduplicates(e_uci_appgroups): + first = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], [], ['Dropbox']) + second = dpi.add_appgroup(e_uci_appgroups, 'Group two', ['netify.netflix', 'netify.tesla']) + assert dpi.expand_appgroups(e_uci_appgroups, [first, second]) == ( + "(app == 'netify.netflix' || app == 'netify.tesla' || proto == 'dropbox')" + ) + + +def test_expand_appgroups_skips_unknown_groups(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + assert dpi.expand_appgroups(e_uci_appgroups, [group_id, 'ns_nonexistent', 'rule0']) == ( + "(app == 'netify.netflix')" + ) + + +def test_expand_appgroups_without_groups(e_uci_appgroups): + assert dpi.expand_appgroups(e_uci_appgroups, []) == '' + + +# rules + +@pytest.fixture +def e_uci_rules(e_uci, mock_vocabularies): + """A clean dpi config with one application group to build rules on.""" + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(dpi_minimal_db) + return e_uci + + +def group_of(e_uci, name='Group one'): + return dpi.add_appgroup(e_uci, name, ['netify.netflix']) + + +def test_list_rules_when_empty(e_uci_rules): + assert dpi.list_rules(e_uci_rules) == [] + + +def test_add_rule(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', ['192.168.1.0/24'], [group]) + assert e_uci_rules.get('dpi', rule_id) == 'rule' + assert e_uci_rules.get('dpi', rule_id, 'ns_name') == 'Block streaming' + assert e_uci_rules.get('dpi', rule_id, 'ns_managed') == '1' + assert e_uci_rules.get('dpi', rule_id, 'enabled') == '1' + assert e_uci_rules.get('dpi', rule_id, 'action') == 'block' + assert e_uci_rules.get('dpi', rule_id, 'priority') == '1' + assert e_uci_rules.get('dpi', rule_id, 'source', list=True) == ('192.168.1.0/24',) + assert e_uci_rules.get('dpi', rule_id, 'appgroup', list=True) == (group,) + + +def test_add_rule_without_source(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block everywhere', True, 'block', [], [group]) + assert e_uci_rules.get('dpi', rule_id, 'source', default=None) is None + + +def test_add_rule_enables_the_engine(e_uci_rules): + group = group_of(e_uci_rules) + dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + assert e_uci_rules.get('dpi', 'config', 'enabled') == '1' + + +def test_add_rule_at_the_top(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + second = dpi.add_rule(e_uci_rules, 'Second', True, 'allow', [], [group], position='top') + assert [rule['id'] for rule in dpi.list_rules(e_uci_rules)] == [second, first] + assert e_uci_rules.get('dpi', second, 'priority') == '1' + assert e_uci_rules.get('dpi', first, 'priority') == '2' + + +def test_add_rule_requires_a_group(e_uci_rules): + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'No match', True, 'block', ['192.168.1.1'], []) + assert err.value.args[0] == 'appgroups' + assert err.value.args[1] == 'appgroups_required' + + +def test_add_rule_refuses_an_unknown_group(e_uci_rules): + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'Bad group', True, 'block', [], ['ns_nonexistent']) + assert err.value.args[1] == 'appgroup_does_not_exists' + + +def test_add_rule_refuses_an_unknown_action(e_uci_rules): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'Bad action', True, 'bulk', [], [group]) + assert err.value.args[0] == 'action' + assert err.value.args[1] == 'invalid_action' + + +def test_add_rule_requires_a_name(e_uci_rules): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, ' ', True, 'block', [], [group]) + assert err.value.args[1] == 'name_required' + + +@pytest.mark.parametrize('source', ['192.168.1.1', '192.168.1.0/24', '192.168.1.10-192.168.1.20', + '2001:db8::1', '2001:db8::/64']) +def test_add_rule_accepts_every_source_shape(e_uci_rules, source): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, f'Rule {source}', True, 'block', [source], [group]) + assert e_uci_rules.get('dpi', rule_id, 'source', list=True) == (source,) + + +@pytest.mark.parametrize('source', ['not-an-ip', '192.168.1.300', '192.168.1.20-192.168.1.10', + '192.168.1.1-2001:db8::1', '']) +def test_add_rule_refuses_an_invalid_source(e_uci_rules, source): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'Bad source', True, 'block', [source], [group]) + assert err.value.args[0] == 'source' + assert err.value.args[1] == 'invalid_source' + + +def test_edit_rule(e_uci_rules): + group = group_of(e_uci_rules) + other = group_of(e_uci_rules, 'Group two') + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', ['192.168.1.1'], [group]) + dpi.edit_rule(e_uci_rules, rule_id, 'Allow streaming', False, 'allow', [], [other]) + assert e_uci_rules.get('dpi', rule_id, 'ns_name') == 'Allow streaming' + assert e_uci_rules.get('dpi', rule_id, 'enabled') == '0' + assert e_uci_rules.get('dpi', rule_id, 'action') == 'allow' + assert e_uci_rules.get('dpi', rule_id, 'appgroup', list=True) == (other,) + # an emptied source is removed, the rule matches every host again + assert e_uci_rules.get('dpi', rule_id, 'source', default=None) is None + + +def test_edit_rule_keeps_the_priority(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + second = dpi.add_rule(e_uci_rules, 'Second', True, 'block', [], [group]) + dpi.edit_rule(e_uci_rules, first, 'First edited', True, 'block', [], [group]) + assert [rule['id'] for rule in dpi.list_rules(e_uci_rules)] == [first, second] + + +def test_edit_rule_of_unknown_id(e_uci_rules): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.edit_rule(e_uci_rules, 'ns_nonexistent', 'Rule', True, 'block', [], [group]) + assert err.value.args[1] == 'rule_not_found' + + +def test_edit_rule_refuses_an_unmanaged_rule(e_uci_with_data, mock_vocabularies): + group = dpi.add_appgroup(e_uci_with_data, 'Group one', ['netify.netflix']) + with pytest.raises(ValidationError) as err: + dpi.edit_rule(e_uci_with_data, 'rule0', 'Rule', True, 'block', [], [group]) + assert err.value.args[1] == 'rule_not_managed' + + +def test_delete_rule_closes_the_gap(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + second = dpi.add_rule(e_uci_rules, 'Second', True, 'block', [], [group]) + third = dpi.add_rule(e_uci_rules, 'Third', True, 'block', [], [group]) + dpi.delete_rule(e_uci_rules, second) + assert e_uci_rules.get('dpi', first, 'priority') == '1' + assert e_uci_rules.get('dpi', third, 'priority') == '2' + + +def test_add_rule_leaves_the_change_pending(e_uci_rules, mock_apply): + group = group_of(e_uci_rules) + mock_apply.reset_mock() + dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + mock_apply.assert_not_called() + + +def test_delete_rule_leaves_the_change_pending(e_uci_rules, mock_apply): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + mock_apply.reset_mock() + dpi.delete_rule(e_uci_rules, rule_id) + mock_apply.assert_not_called() + + +def test_delete_rule_of_unknown_id(e_uci_rules): + with pytest.raises(ValidationError) as err: + dpi.delete_rule(e_uci_rules, 'ns_nonexistent') + assert err.value.args[1] == 'rule_not_found' + + +def test_delete_last_enabled_rule_disables_the_engine(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Only one', True, 'block', [], [group]) + dpi.delete_rule(e_uci_rules, rule_id) + assert e_uci_rules.get('dpi', 'config', 'enabled') == '0' + + +def test_rename_and_toggle_rule(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + dpi.rename_rule(e_uci_rules, rule_id, 'Renamed') + assert e_uci_rules.get('dpi', rule_id, 'ns_name') == 'Renamed' + dpi.disable_rule(e_uci_rules, rule_id) + assert e_uci_rules.get('dpi', rule_id, 'enabled') == '0' + assert e_uci_rules.get('dpi', 'config', 'enabled') == '0' + dpi.enable_rule(e_uci_rules, rule_id) + assert e_uci_rules.get('dpi', rule_id, 'enabled') == '1' + assert e_uci_rules.get('dpi', 'config', 'enabled') == '1' + + +def test_rename_and_toggle_work_on_unmanaged_rules(e_uci_with_data): + dpi.rename_rule(e_uci_with_data, 'rule0', 'Migrated rule') + assert e_uci_with_data.get('dpi', 'rule0', 'ns_name') == 'Migrated rule' + dpi.disable_rule(e_uci_with_data, 'rule0') + assert e_uci_with_data.get('dpi', 'rule0', 'enabled') == '0' + + +def test_order_rules(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + second = dpi.add_rule(e_uci_rules, 'Second', True, 'block', [], [group]) + third = dpi.add_rule(e_uci_rules, 'Third', True, 'block', [], [group]) + dpi.order_rules(e_uci_rules, [third, first, second]) + assert [rule['id'] for rule in dpi.list_rules(e_uci_rules)] == [third, first, second] + assert e_uci_rules.get('dpi', third, 'priority') == '1' + + +def test_order_rules_refuses_a_partial_order(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + dpi.add_rule(e_uci_rules, 'Second', True, 'block', [], [group]) + with pytest.raises(ValidationError) as err: + dpi.order_rules(e_uci_rules, [first]) + assert err.value.args[1] == 'invalid_order' -def test_list_exemptions(e_uci_with_data): - assert dpi.list_exemptions(e_uci_with_data) == [ - { - 'config-name': 'exemp1', - 'enabled': True, - 'criteria': '192.168.1.1', - 'description': 'my host', - } - ] -def test_add_exemption(e_uci): - ex_created = dpi.add_exemption(e_uci, "192.168.2.2", 'my host2', True) - assert dpi.list_exemptions(e_uci) == [ +def test_order_rules_refuses_an_unknown_rule(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + with pytest.raises(ValidationError) as err: + dpi.order_rules(e_uci_rules, [first, 'ns_nonexistent']) + assert err.value.args[1] == 'rule_not_found' + + +def test_list_rules(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', ['192.168.1.0/24'], [group]) + assert dpi.list_rules(e_uci_rules) == [ { - 'config-name': ex_created, + 'id': rule_id, + 'name': 'Block streaming', 'enabled': True, - 'criteria': "192.168.2.2", - 'description': 'my host2', + 'action': 'block', + 'source': ['192.168.1.0/24'], + 'appgroups': [{'id': group, 'name': 'Group one'}], + 'managed': True, + 'index': 0 } ] + + +def test_list_rules_reports_unmanaged_rules(e_uci_with_data): + rules = {rule['id']: rule for rule in dpi.list_rules(e_uci_with_data)} + assert rules['rule0']['managed'] is False + assert rules['rule2']['criteria'] == 'local_ip == 192.168.100.22 && application == "netify.facebook";' + assert 'criteria' not in rules['rule3'] + + +def test_list_rules_hides_invisible_ones(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Hidden', True, 'block', [], [group]) + e_uci_rules.set('dpi', rule_id, 'ns_visible', '0') + assert dpi.list_rules(e_uci_rules) == [] + + +def test_list_rules_is_complete_and_indexed(e_uci_rules): + group = group_of(e_uci_rules) + for index in range(5): + dpi.add_rule(e_uci_rules, f'Rule {index}', True, 'block', [], [group]) + rules = dpi.list_rules(e_uci_rules) + assert [rule['name'] for rule in rules] == ['Rule 0', 'Rule 1', 'Rule 2', 'Rule 3', 'Rule 4'] + assert [rule['index'] for rule in rules] == [0, 1, 2, 3, 4] + + +def test_expand_source(): + assert dpi.expand_source(['192.168.1.1', '192.168.1.0/24']) == ['192.168.1.1', '192.168.1.0/24'] + assert dpi.expand_source(['192.168.1.10-192.168.1.11']) == ['192.168.1.10/31'] + assert dpi.expand_source(['192.168.1.1-192.168.1.4']) == ['192.168.1.1/32', '192.168.1.2/31', + '192.168.1.4/32'] + assert dpi.expand_source(['10.0.0.1-10.0.0.2']) == ['10.0.0.1/32', '10.0.0.2/32'] + + +def test_build_rule_criteria(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', + ['192.168.1.0/24', '10.0.0.1-10.0.0.2'], [group]) + rule = e_uci_rules.get_all('dpi', rule_id) + assert dpi.build_rule_criteria(e_uci_rules, rule) == ( + "(local_ip == 192.168.1.0/24 || local_ip == 10.0.0.1/32 || local_ip == 10.0.0.2/32)" + " && (app == 'netify.netflix');" + ) + + +def test_build_rule_criteria_without_source(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + rule = e_uci_rules.get_all('dpi', rule_id) + assert dpi.build_rule_criteria(e_uci_rules, rule) == "(app == 'netify.netflix');" + + +def test_build_rule_criteria_of_a_rule_matching_nothing(e_uci_rules): + assert dpi.build_rule_criteria(e_uci_rules, {'source': ['192.168.1.1']}) == '' + + +def test_build_rule_criteria_source_only_when_appgroup_not_required(e_uci_rules): + rule = {'source': ['192.168.1.1', '192.168.1.0/24']} + assert dpi.build_rule_criteria(e_uci_rules, rule, require_appgroup=False) == ( + '(local_ip == 192.168.1.1 || local_ip == 192.168.1.0/24);' + ) + + +def test_build_rule_criteria_still_empty_without_source_or_appgroup(e_uci_rules): + assert dpi.build_rule_criteria(e_uci_rules, {}, require_appgroup=False) == '' + + +def test_add_rule_without_appgroup_requires_the_flag(e_uci_rules): with pytest.raises(ValidationError) as err: - dpi.add_exemption(e_uci, "192.168.2.2", 'duplicated', True) + dpi.add_rule(e_uci_rules, 'No group', True, 'allow', ['192.168.1.1'], []) + assert err.value.args[1] == 'appgroups_required' + + +def test_add_rule_source_only_via_migration_flag(e_uci_rules): + rule_id = dpi.add_rule(e_uci_rules, 'Migrated exception 1', True, 'allow', ['192.168.1.1'], [], + require_appgroup=False) + assert e_uci_rules.get('dpi', rule_id, 'ns_managed') == '1' + assert e_uci_rules.get('dpi', rule_id, 'source', list=True) == ('192.168.1.1',) + assert e_uci_rules.get('dpi', rule_id, 'appgroup', default=None) is None + + +# Migration from the schema used before application groups existed + +def test_freeze_legacy_criteria_from_application_and_device(e_uci: EUci): + rule = {'device': 'eth0', 'application': ['netify.amazon-prime', 'netify.netflix']} + assert dpi.freeze_legacy_criteria(e_uci, rule) == ( + "(iface_nfq_src == 'eth0' or iface_nfq_dst == 'eth0') && " + "(app == 'netify.amazon-prime' or app == 'netify.netflix') ;" + ) + + +def test_freeze_legacy_criteria_prefers_hand_written_criteria(e_uci: EUci): + rule = {'device': 'eth0', 'criteria': 'app == "netify.netflix";', 'application': ['netify.amazon-prime']} + assert dpi.freeze_legacy_criteria(e_uci, rule) == "app == 'netify.netflix';" + + +def test_freeze_legacy_criteria_applies_vlan_rewrite(e_uci: EUci): + with pathlib.Path(e_uci.confdir()).joinpath('network').open('a') as fp: + fp.write(""" +config device + option name 'eth0.10' + option ifname 'eth0' + option type '8021q' + option vid '10' +""") + rule = {'device': 'eth0.10', 'application': ['netify.netflix']} + assert dpi.freeze_legacy_criteria(e_uci, rule) == ( + "vlan_id == 10 && (iface_nfq_src == 'eth0.10' or iface_nfq_dst == 'eth0.10') && " + "(app == 'netify.netflix') ;" + ) + + +legacy_dpi_db = """ +config main 'config' + option log_blocked '0' + option firewall_exemption '1' + option enabled '0' + list popular_filters 'netify.facebook' + +config rule 'ns_b01a0e73' + option enabled '1' + option device 'eth0' + option action 'block' + list application 'netify.amazon-prime' + list application 'netify.netflix' + +config rule 'ns_qos1' + option enabled '1' + option device 'eth1' + option action 'bulk' + list application 'netify.dropbox' + +config rule 'ns_empty1' + option enabled '1' + option device 'eth2' + option action 'block' + +config rule 'ns_perrule_exempt' + option enabled '1' + option device 'eth3' + option action 'block' + list application 'netify.tesla' + list exemption '192.168.100.3' + +config exemption 'ns_2127b876' + option enabled '1' + option criteria '192.168.122.47' + option description 'my exception' + +config exemption 'ns_disabled_exempt' + option enabled '0' + option criteria '10.0.0.5' + option description 'disabled one' + +config exemption 'ns_bad_exempt' + option enabled '1' + option criteria 'not-an-address' + option description '' +""" + + +@pytest.fixture +def e_uci_legacy(e_uci: EUci): + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(legacy_dpi_db) + return e_uci + + +def test_migrate_schema_noop_when_nothing_to_migrate(e_uci, mock_apply): + assert dpi.migrate_schema(e_uci) is False + mock_apply.assert_not_called() + + +def test_migrate_schema_freezes_legacy_rule(e_uci_legacy): + assert dpi.migrate_schema(e_uci_legacy) is True + rule = e_uci_legacy.get_all('dpi', 'ns_b01a0e73') + assert rule['criteria'] == ( + "(iface_nfq_src == 'eth0' or iface_nfq_dst == 'eth0') && " + "(app == 'netify.amazon-prime' or app == 'netify.netflix') ;" + ) + assert 'device' not in rule + assert 'application' not in rule + assert 'ns_managed' not in rule + assert rule['action'] == 'block' + assert 'priority' in rule + + +def test_migrate_schema_names_migrated_rules_and_exemptions_in_order(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + assert e_uci_legacy.get('dpi', 'ns_b01a0e73', 'ns_name') == 'Migrated rule 1' + # ns_qos1 is dropped (QoS), ns_empty1 is dropped (matches nothing): ns_perrule_exempt is next + perrule_ids = [s for s, r in utils.get_all_by_type(e_uci_legacy, 'dpi', 'rule').items() + if r.get('ns_name') == 'Migrated rule 2'] + assert len(perrule_ids) == 1 + + +def test_migrate_schema_drops_qos_rule(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + assert e_uci_legacy.get('dpi', 'ns_qos1', default=None) is None + + +def test_migrate_schema_drops_rule_matching_nothing(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + assert e_uci_legacy.get('dpi', 'ns_empty1', default=None) is None + + +def test_migrate_schema_drops_per_rule_exemption_but_keeps_the_rule(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rule = e_uci_legacy.get_all('dpi', 'ns_perrule_exempt') + assert 'exemption' not in rule + assert rule['action'] == 'block' + assert 'criteria' in rule + + +def test_migrate_schema_converts_address_exemption_to_managed_allow_rule(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules = dpi.list_rules(e_uci_legacy) + exception1 = next(r for r in rules if r['name'] == 'Migrated exception 1') + assert exception1['managed'] is True + assert exception1['action'] == 'allow' + assert exception1['source'] == ['192.168.122.47'] + assert exception1['appgroups'] == [] + assert exception1['enabled'] is True + + +def test_migrate_schema_keeps_a_disabled_exemption_disabled(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules = dpi.list_rules(e_uci_legacy) + exception2 = next(r for r in rules if r['name'] == 'Migrated exception 2') + assert exception2['enabled'] is False + + +def test_migrate_schema_converts_non_address_exemption_to_unmanaged_allow_rule(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules = dpi.list_rules(e_uci_legacy) + exception3 = next(r for r in rules if r['name'] == 'Migrated exception 3') + assert exception3['managed'] is False + assert exception3['action'] == 'allow' + assert exception3.get('criteria') == 'not-an-address;' + + +def test_migrate_schema_converts_object_exemption_via_expanded_addresses(e_uci, mocker): + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(""" +config main 'config' + option enabled '0' + +config exemption 'ns_objex' + option enabled '1' + option criteria 'objects/ns_obj1' + option description 'office subnet' +""") + mocker.patch('nethsec.dpi.objects.is_object_id', return_value=True) + mocker.patch('nethsec.dpi.objects.get_object_ips', return_value=['192.168.50.0/24']) + assert dpi.migrate_schema(e_uci) is True + rules = dpi.list_rules(e_uci) + assert len(rules) == 1 + assert rules[0]['managed'] is True + assert rules[0]['source'] == ['192.168.50.0/24'] + + +def test_migrate_schema_removes_exemption_sections_and_legacy_globals(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + assert utils.get_all_by_type(e_uci_legacy, 'dpi', 'exemption') == {} + assert e_uci_legacy.get('dpi', 'config', 'firewall_exemption', default=None) is None + assert e_uci_legacy.get('dpi', 'config', 'popular_filters', list=True, default=None) is None + + +def test_migrate_schema_puts_exemptions_before_migrated_rules(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules = dpi.list_rules(e_uci_legacy) + names_in_order = [r['name'] for r in rules] + assert names_in_order.index('Migrated exception 1') < names_in_order.index('Migrated rule 1') + + +def test_migrate_schema_applies_immediately(e_uci_legacy, mock_apply): + assert dpi.migrate_schema(e_uci_legacy) is True + mock_apply.assert_called_with(e_uci_legacy) -def test_edit_exemption(e_uci_with_data): - dpi.edit_exemption(e_uci_with_data, 'exemp1', '192.168.1.3', 'my host 3', False) - assert dpi.list_exemptions(e_uci_with_data) == [ - { - 'config-name': 'exemp1', - 'enabled': False, - 'criteria': "192.168.1.3", - 'description': 'my host 3', - } - ] -def test_delete_exemption(e_uci): - dpi.delete_exemption(e_uci, 'exemp1') - assert dpi.list_exemptions(e_uci) == [] +def test_migrate_schema_is_idempotent(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules_after_first_run = dpi.list_rules(e_uci_legacy) + assert dpi.migrate_schema(e_uci_legacy) is False + assert dpi.list_rules(e_uci_legacy) == rules_after_first_run From 24111788cb870571d91bf6f84ee950336e1e516c Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Tue, 8 Sep 2026 16:31:01 +0200 Subject: [PATCH 02/17] feat(dpi): update UI source version to specific commit --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 7245172c5..e6d682c89 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=$(PKG_VERSION) +PKG_SOURCE_VERSION:=cab1649acd4fc2034d8616a29b4a1fbc4037fa0d PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip From 6d3dca8941808eb8502224967b9e62f8962a375f Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Fri, 18 Sep 2026 10:42:04 +0200 Subject: [PATCH 03/17] feat(dpi): update UI source version to specific commit v2 --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index e6d682c89..8e9d06dfe 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=cab1649acd4fc2034d8616a29b4a1fbc4037fa0d +PKG_SOURCE_VERSION:=f7485d754c30bee6acc7b6f664915b9fe98cc5d2 PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip From e0d164d9cb73187ad74574b33b40e8e5f27e87e6 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Fri, 18 Sep 2026 15:11:53 +0200 Subject: [PATCH 04/17] feat(dpi): update UI source version to specific commit v3 --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 8e9d06dfe..3e42e7b73 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=f7485d754c30bee6acc7b6f664915b9fe98cc5d2 +PKG_SOURCE_VERSION:=a1c0cbc56c85742a8dc1bd5d690063c5ad4f4373 PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip From d3e8847610402575f5f1a6eac8656109cdc3d5bd Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Fri, 18 Sep 2026 16:35:41 +0200 Subject: [PATCH 05/17] feat(dpi): remove unused application and protocol loading functions api --- packages/ns-api/README.md | 81 --- packages/ns-api/files/ns.dpi | 27 +- .../src/nethsec/dpi/__init__.py | 163 ------ packages/python3-nethsec/tests/test_dpi.py | 464 ------------------ 4 files changed, 1 insertion(+), 734 deletions(-) diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index ad809fa2f..97248f3af 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -3550,87 +3550,6 @@ Response example: Manage netifyd DPI engine. -### list-applications - -List application and protocols: - -```bash -api-cli ns.dpi list-applications -``` - -Filtering is provided out of the box, searching in both name and category: - -```bash -api-cli ns.dpi list-applications --data '{"search": "apple"}' -``` - -Data can be limited and paginated by using the `limit` and `page` parameters: - -```bash -api-cli ns.dpi list-applications --data '{"limit": 10, "page": 3}' -``` - -**PLEASE NOTE**: - -- `category` field can be missing in some applications/protocols. -- applications and protocols are the ones loaded in memory by the netifyd engine; if the engine can't be queried, applications are read from the installed signatures file. - -Example response: -```json -{ - "values": { - "data": [ - { - "id": 10392, - "name": "netify.apple-siri", - "type": "application", - "category": { - "name": "business" - } - }, - { - "id": 10706, - "name": "netify.apple-id", - "type": "application" - }, - { - "id": 10152, - "name": "netify.appnexus", - "type": "application", - "category": { - "name": "advertiser" - } - }, - { - "id": 142, - "name": "WhatsApp", - "type": "protocol", - "category": { - "name": "messaging" - } - }, - { - "id": 238, - "name": "Apple/Push", - "type": "protocol" - }, - { - "id": 246, - "name": "WhatsApp/Call", - "type": "protocol", - "category": { - "name": "voip" - } - } - ], - "meta": { - "last_page": 2, - "total": 12 - } - } -} -``` - ### list-loaded-applications List the applications the netifyd engine has loaded in memory, as returned by `netifyd --dump-apps`: diff --git a/packages/ns-api/files/ns.dpi b/packages/ns-api/files/ns.dpi index cac3b9202..a268693e6 100755 --- a/packages/ns-api/files/ns.dpi +++ b/packages/ns-api/files/ns.dpi @@ -15,11 +15,6 @@ cmd = sys.argv[1] if cmd == 'list': print(json.dumps({ - 'list-applications': { - 'search': 'str', - 'limit': 32, - 'page': 32 - }, 'list-rules': {}, 'add-rule': { 'name': 'str', @@ -53,10 +48,8 @@ if cmd == 'list': 'order-rules': { 'order': ['ns_1234', 'ns_5678'] }, - 'list-application-categories': {}, 'list-application-catalog': {}, 'list-loaded-applications': {}, - 'list-protocol-categories': {}, 'list-protocol-catalog': {}, 'list-loaded-protocols': {}, 'list-appgroups': { @@ -87,14 +80,7 @@ elif cmd == 'call': action = sys.argv[2] e_uci = EUci() try: - if action == 'list-application-categories': - try: - with open('/etc/netifyd/netify-application-categories.json', 'r') as f: - content = json.load(f) - print(json.dumps({'values': content})) - except Exception: - print(json.dumps({})) - elif action == 'list-application-catalog': + if action == 'list-application-catalog': try: with open('/etc/netifyd/netify-application-catalog.json', 'r') as f: content = json.load(f) @@ -109,13 +95,6 @@ elif cmd == 'call': else: print(json.dumps({'values': [{'id': app_id, 'name': app_name} for app_id, app_name in applications.items()]})) - elif action == 'list-protocol-categories': - try: - with open('/etc/netifyd/netify-protocol-categories.json', 'r') as f: - content = json.load(f) - print(json.dumps({'values': content})) - except Exception: - print(json.dumps({})) elif action == 'list-protocol-catalog': try: with open('/etc/netifyd/netify-protocol-catalog.json', 'r') as f: @@ -131,10 +110,6 @@ elif cmd == 'call': else: print(json.dumps({'values': [{'id': proto_id, 'name': proto_name} for proto_id, proto_name in protocols.items()]})) - elif action == 'list-applications': - data = json.JSONDecoder().decode(sys.stdin.read()) - result = dpi.list_applications(data.get('search', None), data.get('limit', None), data.get('page', 1)) - print(json.dumps({'values': result})) elif action == 'list-rules': print(json.dumps({'values': dpi.list_rules(e_uci)})) elif action == 'add-rule': diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py index 57cc50cd5..f0900d4e8 100644 --- a/packages/python3-nethsec/src/nethsec/dpi/__init__.py +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -21,22 +21,6 @@ from nethsec.utils import ValidationError -def __load_applications() -> dict[int, str]: - """ - Reads the applications from the netify-apps.conf file. - - Returns: - dict of applications, each dict contains the property "id" and "name" - """ - applications = dict[int, str]() - with open('/etc/netifyd/netify-apps.conf', 'r') as file: - for line in file.readlines(): - if line.startswith('app'): - line_split = line.strip().removesuffix('\n').removeprefix('app:').split(":") - applications[int(line_split[0])] = line_split[1] - return applications - - def load_applications() -> dict[int, str]: """ Reads the applications loaded in memory by the engine, using the netifyd --dump-apps command. @@ -63,39 +47,6 @@ def load_applications() -> dict[int, str]: return applications -def __load_application_categories() -> dict[int, dict[str]]: - """ - Reads the application categories from the netify-categories.json file. - - Returns: - dict of application categories, each dict contains the property "id" and "name" - """ - categories = dict[int, dict[str]]() - with open('/etc/netifyd/netify-categories.json', 'r') as file: - categories_file = json.load(file) - - categories_names = dict[int, str]() - if 'application_tag_index' not in categories_file: - for category_name, applications in categories_file['application_index'].items(): - for application in applications: - categories[application] = { - 'name': category_name - } - else: - categories_application_tag_index: dict[str, int] = categories_file['application_tag_index'] - for category_name, category_id in categories_application_tag_index.items(): - categories_names[category_id] = category_name - - categories_application_index: list[int, list[int]] = categories_file['application_index'] - for category_id, applications_id in categories_application_index: - for application_id in applications_id: - categories[application_id] = { - 'name': categories_names[category_id] - } - - return categories - - def load_protocols() -> dict[int, str]: """ Reads the protocols from the netifyd --dump-protos command. @@ -115,120 +66,6 @@ def load_protocols() -> dict[int, str]: return protocols -def __load_protocol_categories() -> dict[int, dict[str]]: - """ - Reads the protocol categories from the netify-categories.json file. - - Returns: - dict of protocol categories, each dict contains the property "id" and "name" - """ - categories = dict[int, dict[str]]() - with open('/etc/netifyd/netify-categories.json', 'r') as file: - categories_file = json.load(file) - - categories_names = dict[int, str]() - - if 'protocol_tag_index' not in categories_file: - for category_name, protocols in categories_file['protocol_index'].items(): - for protocol in protocols: - categories[protocol] = { - 'name': category_name - } - else: - categories_protocol_tag_index: dict[str, int] = categories_file['protocol_tag_index'] - for category_name, category_id in categories_protocol_tag_index.items(): - categories_names[category_id] = category_name - - categories_protocol_index: list[int, list[int]] = categories_file['protocol_index'] - for category_id, protocol_ids in categories_protocol_index: - for protocol_id in protocol_ids: - categories[protocol_id] = { - 'name': categories_names[category_id] - } - - return categories - - -def __load_blocklist() -> list[dict[str]]: - """ - Format the applications and protocols into a list of dicts. - - Returns: - list of dicts, each dict contains the property "id", "name", "type" and "category" - """ - result = list[dict[str]]() - try: - applications = load_applications() - except Exception: - # the engine can't be queried, fall back to the signatures file - applications = __load_applications() - application_categories = __load_application_categories() - - for application_id, application_name in applications.items(): - result_application = { - 'id': application_id, - 'name': application_name, - 'type': 'application' - } - if application_id in application_categories: - result_application['category'] = application_categories[application_id] - result.append(result_application) - - protocols = load_protocols() - protocol_categories = __load_protocol_categories() - - for protocol_id, protocol_name in protocols.items(): - result_protocol = { - 'id': protocol_id, - 'name': protocol_name, - 'type': 'protocol' - } - if protocol_id in protocol_categories: - result_protocol['category'] = protocol_categories[protocol_id] - result.append(result_protocol) - - return result - - -def list_applications(search: str = None, limit: int = None, page: int = 1) -> dict: - """ - List applications available for filtering. - - Args: - - search: search string - - limit: limit the number of results - - page: page number - - Returns: - list of dicts, each dict contains the property "code" and "name" - """ - result = __load_blocklist() - - if search is not None: - # lower string so we can do a case-insensitive search - search = search.lower() - # I'm aware it's far from a readable code, but list comprehension is the fastest way to filter. - result = [item for item in result if - item.get('name', '').lower().find(search) != -1 or - item.get('category', {}).get('name', '').lower().find(search) != -1] - - total = len(result) - - if limit is not None: - result = result[limit * (page - 1):limit * page] - last_page = math.ceil(total / limit) - else: - last_page = 1 - - return { - 'data': result, - 'meta': { - 'last_page': last_page, - 'total': total, - } - } - - def __apply(e_uci: EUci): """ Commit the dpi config and reload the dpi service. diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py index 2bcdb8072..fddd653e7 100644 --- a/packages/python3-nethsec/tests/test_dpi.py +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -7,27 +7,6 @@ from nethsec import dpi, utils from nethsec.utils import ValidationError -applications_file = """ -# Comment to avoid - -# another comment to avoid - -app:133:netify.netflix -app:10119:netify.linkedin -app:10552:netify.tesla -app:10195:netify.avira -app:10194:netify.sophos -app:10244:netify.bbc -app:10362:netify.hulu -app:10118:netify.lets-encrypt -app:199:netify.snapchat -dom:133:dualstack.apiproxy-device-prod-nlb-1-4d12762d4ba53e45.elb.eu-west-1.amazonaws.com -net:10852:208.118.237.0/24 -net:124:208.117.254.0/24 -nsd:-1:39:YXBwID09ICduZXRpZnkuc2lnbmFsJyAmJiBwcm90b2NvbF9pZCA9PSA3OCAmJiAob3RoZXJfcG9ydCA9PSA0NDMgfHwgbG9jYWxfcG9ydCA9PSA0NDMpOw== -nsd:-1:67:YXBwID09ICduZXRpZnkuZ29vZ2xlLWNoYXQnICYmIHByb3RvY29sX2lkICE9IDY3ICYmIChvdGhlcl9wb3J0ID09IDUyMjggfHwgbG9jYWxfcG9ydCA9PSA1MjI4KTs= -""" - applications = { 133: 'netify.netflix', 10119: 'netify.linkedin', @@ -50,153 +29,6 @@ 130: 'HTTP/Connect' } -categories_file = """ -{ - "application_index": [ - [ - 3, - [ - 10362, - 10194, - 10552 - ] - ], - [ - 33, - [ - 133, - 10119, - 10195, - 10244 - ] - ], - [ - 20, - [ - 199 - ] - ] - ], - "application_tag_index": { - "first-category": 3, - "unknown": 33, - "last": 20 - }, - "protocol_index": [ - [ - 1, - [ - 121, - 127, - 128 - ] - ], - [ - 2, - [ - 116 - ] - ], - [ - 4, - [ - 129, - 130 - ] - ] - ], - "protocol_tag_index": { - "base": 1, - "games": 2, - "low": 4 - } -} -""" - -categories_file_old = """ -{ - "last_update": 1696950440, - "application_index": { - "first-category": [ - 10362, - 10194, - 10552 - ], - "unknown": [ - 133, - 10119, - 10195, - 10244 - ], - "last": [ - 199 - ] - }, - "protocol_index": { - "base": [ - 121, - 127, - 128 - ], - "games": [ - 116 - ], - "low": [ - 129, - 130 - ] - } -} -""" - -application_categories = { - 10362: { - 'name': 'first-category' - }, - 10194: { - 'name': 'first-category' - }, - 10552: { - 'name': 'first-category' - }, - 133: { - 'name': 'unknown' - }, - 10119: { - 'name': 'unknown' - }, - 10195: { - 'name': 'unknown' - }, - 10244: { - 'name': 'unknown' - }, - 199: { - 'name': 'last' - } -} - -protocol_categories = { - 121: { - 'name': 'base' - }, - 127: { - 'name': 'base' - }, - 128: { - 'name': 'base' - }, - 116: { - 'name': 'games' - }, - 129: { - 'name': 'low' - }, - 130: { - 'name': 'low' - } -} - application_output = """ 133: netify.netflix 10119: netify.linkedin @@ -389,32 +221,6 @@ def e_uci_with_data(e_uci: EUci): return e_uci -@pytest.fixture -def mock_load(mocker): - mocker.patch('nethsec.dpi.__load_applications', return_value=applications) - mocker.patch('nethsec.dpi.load_applications', return_value=applications) - mocker.patch('nethsec.dpi.__load_application_categories', return_value=application_categories) - mocker.patch('nethsec.dpi.load_protocols', return_value=protocols) - mocker.patch('nethsec.dpi.__load_protocol_categories', return_value=protocol_categories) - - -def test_load_applications(mocker: MockFixture): - mocker.patch('builtins.open', mocker.mock_open(read_data=applications_file)) - assert dpi.__load_applications() == applications - - -@pytest.mark.parametrize('data', [categories_file, categories_file_old]) -def test_load_application_categories(mocker: MockFixture, data: str): - mocker.patch('builtins.open', mocker.mock_open(read_data=data)) - assert dpi.__load_application_categories() == application_categories - - -@pytest.mark.parametrize('data', [categories_file, categories_file_old]) -def test_load_protocol_categories(mocker: MockFixture, data: str): - mocker.patch('builtins.open', mocker.mock_open(read_data=data)) - assert dpi.__load_protocol_categories() == protocol_categories - - def test_load_applications_from_engine(mocker: MockFixture): process_result = mocker.stub('subprocess_return') process_result.stdout = bytes(application_output, 'utf-8') @@ -429,276 +235,6 @@ def test_load_protocols(mocker: MockFixture): assert dpi.load_protocols() == protocols -@pytest.mark.parametrize('search', [None, '']) -def test_index_applications(mock_load, search): - assert dpi.list_applications(search) == { - 'data': [ - { - 'id': 133, - 'name': 'netify.netflix', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 10119, - 'name': 'netify.linkedin', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 10552, - 'name': 'netify.tesla', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - }, - { - 'id': 10195, - 'name': 'netify.avira', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 10194, - 'name': 'netify.sophos', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - }, - { - 'id': 10244, - 'name': 'netify.bbc', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 10362, - 'name': 'netify.hulu', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - }, - { - 'id': 10118, - 'name': 'netify.lets-encrypt', - 'type': 'application' - }, - { - 'id': 199, - 'name': 'netify.snapchat', - 'type': 'application', - 'category': { - 'name': 'last' - } - }, - { - 'id': 116, - 'name': 'Warcraft3', - 'type': 'protocol', - 'category': { - 'name': 'games' - } - }, - { - 'id': 117, - 'name': 'LotusNotes', - 'type': 'protocol' - }, - { - 'id': 121, - 'name': 'Dropbox', - 'type': 'protocol', - 'category': { - 'name': 'base' - } - }, - { - 'id': 127, - 'name': 'RPC', - 'type': 'protocol', - 'category': { - 'name': 'base' - } - }, - { - 'id': 128, - 'name': 'NetFlow', - 'type': 'protocol', - 'category': { - 'name': 'base' - } - }, - { - 'id': 129, - 'name': 'SFlow', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - }, - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ], - 'meta': { - 'last_page': 1, - 'total': 16 - } - } - - -def test_index_applications_search(mock_load): - assert dpi.list_applications(search='l') == { - 'data': [ - { - 'id': 133, - 'name': 'netify.netflix', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 10119, - 'name': 'netify.linkedin', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 10552, - 'name': 'netify.tesla', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - }, - { - 'id': 10362, - 'name': 'netify.hulu', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - }, - { - 'id': 10118, - 'name': 'netify.lets-encrypt', - 'type': 'application' - }, - { - 'id': 199, - 'name': 'netify.snapchat', - 'type': 'application', - 'category': { - 'name': 'last' - } - }, - { - 'id': 117, - 'name': 'LotusNotes', - 'type': 'protocol' - }, - { - 'id': 128, - 'name': 'NetFlow', - 'type': 'protocol', - 'category': { - 'name': 'base' - } - }, - { - 'id': 129, - 'name': 'SFlow', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - }, - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ], - 'meta': { - 'last_page': 1, - 'total': 10 - } - } - - -def test_index_applications_paginate(mock_load): - assert dpi.list_applications(limit=2, page=2) == { - 'data': [ - { - 'id': 10552, - 'name': 'netify.tesla', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - }, - { - 'id': 10195, - 'name': 'netify.avira', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - } - ], - 'meta': { - 'last_page': 8, - 'total': 16 - } - } - assert dpi.list_applications(limit=2, page=3) == { - 'data': [ - { - 'id': 10194, - 'name': 'netify.sophos', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - }, - { - 'id': 10244, - 'name': 'netify.bbc', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - } - ], - 'meta': { - 'last_page': 8, - 'total': 16 - } - } - - # application groups category_output = """ From b4ef6609c5487810d4bd41d1f1ac5a9c92c6232e Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Tue, 22 Sep 2026 15:39:41 +0200 Subject: [PATCH 06/17] feat(dpi): add support for match_all rule in DPI configuration and API --- packages/ns-api/README.md | 32 ++++++-- packages/ns-api/files/ns.dpi | 9 ++- packages/ns-dpi/files/dpi-config | 5 +- .../src/nethsec/dpi/__init__.py | 73 +++++++++++-------- packages/python3-nethsec/tests/test_dpi.py | 70 +++++++++++++++--- 5 files changed, 136 insertions(+), 53 deletions(-) diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 97248f3af..9b5e3a156 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -3767,7 +3767,7 @@ api-cli ns.dpi list-rules The list is never paginated nor filtered: it is the whole set, which is also what `order-rules` needs. Rules hidden with `ns_visible '0'` are the only ones left out. `managed` tells whether the rule was created through the API: an unmanaged rule can be renamed, enabled, reordered and deleted, but not -edited, and it carries the raw `criteria` it matches on. +edited, and it carries the raw `criteria` it matches on. `match_all` tells whether the rule matches every flow instead of naming application groups. Example response: @@ -3786,9 +3786,21 @@ Example response: "name": "Streaming services" } ], + "match_all": false, "managed": true, "index": 0 }, + { + "id": "ns_9d40be71", + "name": "Allow everything", + "enabled": true, + "action": "allow", + "source": [], + "appgroups": [], + "match_all": true, + "managed": true, + "index": 1 + }, { "id": "ns_f1c6e9e0", "name": "Legacy rule", @@ -3796,9 +3808,10 @@ Example response: "action": "block", "source": [], "appgroups": [], + "match_all": false, "managed": false, "criteria": "local_ip == 192.168.100.22 && app == 'netify.facebook';", - "index": 1 + "index": 2 } ] } @@ -3815,6 +3828,7 @@ api-cli ns.dpi add-rule --data '{ "action": "block", "source": ["192.168.1.0/24"], "appgroups": ["ns_1a2b3c4d5"], + "match_all": false, "position": "bottom" }' ``` @@ -3829,12 +3843,19 @@ Parameters: `["192.168.1.1", "192.168.1.0/24", "10.0.0.10-10.0.0.20"]`. An empty list matches every host. Ranges are expanded to CIDR blocks when the rule is generated - `appgroups`: config names of the application groups the rule matches, refer to `list-appgroups`. **At - least one is required**: a rule with a source and no group would be an IP-level rule, which the - firewall does better, and a rule with neither would match nothing + least one is required unless `match_all` is set**: a rule with a source and no group would be an + IP-level rule, which the firewall does better, and a rule with neither would match nothing +- `match_all`: `true` makes the rule match every flow, whatever the application. `appgroups` must then + be empty, and passing both is refused with `appgroups_not_allowed_with_match_all`. Combined with a + `source`, it means every flow of those hosts. Defaults to `false` - `position`: `top` to evaluate the rule before every other one, `bottom` (the default) after them There is no device or interface parameter: per-interface rules are not reachable from the API. +A `match_all` rule placed at the top of the list shadows every rule below it, because every rule halts +on match. That is the point of an `allow` one — it is how a bypass is expressed — but a client offering +it should make the consequence visible. + Example response: ```json @@ -3855,7 +3876,8 @@ api-cli ns.dpi edit-rule --data '{ "enabled": false, "action": "block", "source": [], - "appgroups": ["ns_1a2b3c4d5"] + "appgroups": ["ns_1a2b3c4d5"], + "match_all": false }' ``` diff --git a/packages/ns-api/files/ns.dpi b/packages/ns-api/files/ns.dpi index a268693e6..db9dbaca9 100755 --- a/packages/ns-api/files/ns.dpi +++ b/packages/ns-api/files/ns.dpi @@ -22,6 +22,7 @@ if cmd == 'list': 'action': 'str', 'source': [], 'appgroups': [], + 'match_all': False, 'position': 'str' }, 'edit-rule': { @@ -30,7 +31,8 @@ if cmd == 'list': 'enabled': False, 'action': 'str', 'source': [], - 'appgroups': [] + 'appgroups': [], + 'match_all': False }, 'delete-rule': { 'id': 'str' @@ -116,12 +118,13 @@ elif cmd == 'call': data = json.JSONDecoder().decode(sys.stdin.read()) config_name = dpi.add_rule(e_uci, data['name'], data['enabled'], data['action'], data.get('source', []), data.get('appgroups', []), - data.get('position', 'bottom')) + data.get('position', 'bottom'), data.get('match_all', False)) print(json.dumps({'id': config_name})) elif action == 'edit-rule': data = json.JSONDecoder().decode(sys.stdin.read()) config_name = dpi.edit_rule(e_uci, data['id'], data['name'], data['enabled'], data['action'], - data.get('source', []), data.get('appgroups', [])) + data.get('source', []), data.get('appgroups', []), + data.get('match_all', False)) print(json.dumps({'id': config_name})) elif action == 'delete-rule': data = json.JSONDecoder().decode(sys.stdin.read()) diff --git a/packages/ns-dpi/files/dpi-config b/packages/ns-dpi/files/dpi-config index 1c114cd08..f5bc79e6a 100755 --- a/packages/ns-dpi/files/dpi-config +++ b/packages/ns-dpi/files/dpi-config @@ -119,12 +119,11 @@ for section in u.get_all('dpi'): is_legacy = any(field in rule for field in legacy_fields) if not is_legacy and 'criteria' not in rule: - # current schema: application groups, optionally narrowed to a source, or a rule migrated from - # a global exemption (source only, no group — dpi.migrate_schema) + # current schema: application groups, optionally narrowed to a source, or a match-all rule if rule.get('action') not in dpi.DPI_RULE_ACTIONS: syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, unknown action {rule.get('action')}") continue - criteria = dpi.build_rule_criteria(u, rule, require_appgroup=False) + criteria = dpi.build_rule_criteria(u, rule) if not criteria: # never emit an empty criteria: it is a parse error, and a parse error kills the agent syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, it matches nothing") diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py index f0900d4e8..ae1b14d31 100644 --- a/packages/python3-nethsec/src/nethsec/dpi/__init__.py +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -494,6 +494,8 @@ def expand_appgroups(e_uci: EUci, config_names: list[str]) -> str: DPI_RULE_NAME_MAX_LENGTH = 64 +DPI_MATCH_ALL_CRITERIA = '*' + DPI_RULE_POSITIONS = ('top', 'bottom') @@ -559,14 +561,14 @@ def __validate_source(values: list[str]) -> list[str]: return sources -def __validate_appgroups(e_uci: EUci, config_names: list[str], require_appgroup: bool = True) -> list[str]: +def __validate_appgroups(e_uci: EUci, config_names: list[str], match_all: bool = False) -> list[str]: """ - Validate the groups a rule matches. At least one is required by default: a rule with a source and - no group would be an IP-level rule, which the firewall does better, and a rule with neither would - expand to an empty criteria. + Validate the groups a rule matches. At least one is required: a rule with a source and no group + would be an IP-level rule, which the firewall does better, and a rule with neither would expand to + an empty criteria. - `require_appgroup=False` lifts that requirement for a rule migrated from a global exemption: it is - a source-only Allow rule by design, a shape the drawer cannot produce and never will. + `match_all` is the way out of that requirement, and the only one: a rule that matches every flow + names no group by definition, so naming one at the same time is a contradiction, not a narrowing. """ appgroups = [] for config_name in config_names or []: @@ -574,7 +576,9 @@ def __validate_appgroups(e_uci: EUci, config_names: list[str], require_appgroup: raise ValidationError('appgroups', 'appgroup_does_not_exists', config_name) if config_name not in appgroups: appgroups.append(config_name) - if require_appgroup and not appgroups: + if match_all and appgroups: + raise ValidationError('appgroups', 'appgroups_not_allowed_with_match_all', appgroups) + if not match_all and not appgroups: raise ValidationError('appgroups', 'appgroups_required', appgroups) return appgroups @@ -600,27 +604,23 @@ def expand_source(values: list[str]) -> list[str]: return expanded -def build_rule_criteria(e_uci: EUci, rule: dict, require_appgroup: bool = True) -> str: +def build_rule_criteria(e_uci: EUci, rule: dict) -> str: """ Build the criteria of a managed rule: the source, when set, ANDed with the union of its groups. Args: - e_uci: euci instance - rule: the rule section, as returned by `uci.get_all` - - require_appgroup: when False, a rule with a source and no group emits a source-only criteria - instead of being treated as matching nothing. Used for a rule migrated from a global exemption, - the one shape of source-only Allow rule the drawer cannot produce and never will. Returns: - the semicolon-terminated expression, or an empty string when the rule matches nothing and must - not be emitted + the expression, or an empty string when the rule matches nothing and must not be emitted """ match = expand_appgroups(e_uci, list(rule.get('appgroup', []))) sources = [f'local_ip == {source}' for source in expand_source(list(rule.get('source', [])))] if not match: - if require_appgroup or not sources: - return '' + if not sources: + return DPI_MATCH_ALL_CRITERIA if rule.get('ns_match_all', '0') == '1' else '' return f"({' || '.join(sources)});" if sources: @@ -639,7 +639,8 @@ def list_rules(e_uci: EUci) -> list[dict]: Returns: list of dicts, each dict contains the property "id", "name", "enabled", "action", "source", - "appgroups", "managed" and "index", plus "criteria" for the rules the API did not create + "appgroups", "match_all", "managed" and "index", plus "criteria" for the rules the API did not + create """ groups = {section: group.get('ns_name', '') for section, group in (utils.get_all_by_type(e_uci, 'dpi', 'appgroup') or {}).items()} @@ -655,6 +656,7 @@ def list_rules(e_uci: EUci) -> list[dict]: 'source': list(rule.get('source', [])), 'appgroups': [{'id': group, 'name': groups.get(group, group)} for group in rule.get('appgroup', [])], + 'match_all': rule.get('ns_match_all', '0') == '1', # a rule the UI did not create can be renamed, toggled, reordered and deleted, but not edited 'managed': rule.get('ns_managed', '0') == '1', 'index': len(rules) @@ -667,15 +669,19 @@ def list_rules(e_uci: EUci) -> list[dict]: def __save_rule_data(e_uci: EUci, config_name: str, name: str, enabled: bool, action: str, - source: list[str], appgroups: list[str]): + source: list[str], appgroups: list[str], match_all: bool = False): e_uci.set('dpi', config_name, 'ns_name', name) e_uci.set('dpi', config_name, 'ns_managed', '1') e_uci.set('dpi', config_name, 'enabled', enabled) e_uci.set('dpi', config_name, 'action', action) + if match_all: + e_uci.set('dpi', config_name, 'ns_match_all', '1') + else: + e_uci.delete('dpi', config_name, 'ns_match_all') if appgroups: e_uci.set('dpi', config_name, 'appgroup', appgroups) else: - # a rule migrated from a global exemption has no group: the option must go, not stay empty + # a match-all rule or a rule migrated from a global exemption has no group: the option must go, not stay empty e_uci.delete('dpi', config_name, 'appgroup') if source: e_uci.set('dpi', config_name, 'source', source) @@ -685,7 +691,7 @@ def __save_rule_data(e_uci: EUci, config_name: str, name: str, enabled: bool, ac def add_rule(e_uci: EUci, name: str, enabled: bool, action: str, source: list[str], - appgroups: list[str], position: str = 'bottom', require_appgroup: bool = True) -> str: + appgroups: list[str], position: str = 'bottom', match_all: bool = False) -> str: """ Store a new rule. @@ -696,10 +702,10 @@ def add_rule(e_uci: EUci, name: str, enabled: bool, action: str, source: list[st - action: 'block' or 'allow' - source: list of addresses, networks or ranges, empty to match every host - appgroups: config names of the application groups the rule matches, at least one unless - `require_appgroup` is False + `match_all` is set - position: 'top' to evaluate the rule before every other one, 'bottom' after them - - require_appgroup: set to False only when migrating a global exemption into a source-only Allow - rule, a shape the drawer cannot produce and never will + - match_all: the rule matches every flow, narrowed to `source` when one is given. Mutually + exclusive with `appgroups` Returns: config name of the rule created @@ -713,11 +719,11 @@ def add_rule(e_uci: EUci, name: str, enabled: bool, action: str, source: list[st if position not in DPI_RULE_POSITIONS: raise ValidationError('position', 'invalid_position', position) source = __validate_source(source) - appgroups = __validate_appgroups(e_uci, appgroups, require_appgroup) + appgroups = __validate_appgroups(e_uci, appgroups, match_all) config_name = utils.get_random_id() e_uci.set('dpi', config_name, 'rule') - __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups) + __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups, match_all) order = [section for section, _ in __sorted_rules(e_uci) if section != config_name] order.insert(0, config_name) if position == 'top' else order.append(config_name) @@ -729,7 +735,7 @@ def add_rule(e_uci: EUci, name: str, enabled: bool, action: str, source: list[st def edit_rule(e_uci: EUci, config_name: str, name: str, enabled: bool, action: str, source: list[str], - appgroups: list[str]) -> str: + appgroups: list[str], match_all: bool = False) -> str: """ Edit a rule. Only rules created through the API can be edited: a rule carrying a hand-written criteria has no source and no group to fill the form with, and rewriting it would change what it @@ -742,7 +748,10 @@ def edit_rule(e_uci: EUci, config_name: str, name: str, enabled: bool, action: s - enabled: enable the rule - action: 'block' or 'allow' - source: list of addresses, networks or ranges, empty to match every host - - appgroups: config names of the application groups the rule matches, at least one + - appgroups: config names of the application groups the rule matches, at least one unless + `match_all` is set + - match_all: the rule matches every flow, narrowed to `source` when one is given. Mutually + exclusive with `appgroups` Returns: config name of the rule edited @@ -759,9 +768,9 @@ def edit_rule(e_uci: EUci, config_name: str, name: str, enabled: bool, action: s if action not in DPI_RULE_ACTIONS: raise ValidationError('action', 'invalid_action', action) source = __validate_source(source) - appgroups = __validate_appgroups(e_uci, appgroups) + appgroups = __validate_appgroups(e_uci, appgroups, match_all) - __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups) + __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups, match_all) __toggle_engine(e_uci) e_uci.save('dpi') return config_name @@ -984,9 +993,9 @@ def migrate_schema(e_uci: EUci) -> bool: action is a retired QoS value, carries a per-rule exemption (a field no recent API ever wrote), or ends up matching nothing is dropped instead, with a log line naming it. - Each global exemption becomes an Allow rule at the top of the list: a managed, source-only rule - (`require_appgroup=False`) when its criteria is a plain address, CIDR or firewall object; an - unmanaged one carrying the criteria verbatim otherwise. Disabled exemptions become disabled rules. + Each global exemption becomes an Allow rule at the top of the list: a managed match-all rule + narrowed to a source when its criteria is a plain address, CIDR or firewall object; an unmanaged + one carrying the criteria verbatim otherwise. Disabled exemptions become disabled rules. The `exemption` section type, `firewall_exemption` and `popular_filters` are then removed. Safe to call unconditionally on every boot: a box with nothing left in the old schema returns False @@ -1019,7 +1028,7 @@ def migrate_schema(e_uci: EUci) -> bool: e_uci.delete('dpi', section) if source is not None: - rule_id = add_rule(e_uci, name, enabled, 'allow', source, [], require_appgroup=False) + rule_id = add_rule(e_uci, name, enabled, 'allow', source, [], match_all=True) else: rule_id = utils.get_random_id() e_uci.set('dpi', rule_id, 'rule') diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py index fddd653e7..c4e35cf04 100644 --- a/packages/python3-nethsec/tests/test_dpi.py +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -749,6 +749,7 @@ def test_list_rules(e_uci_rules): 'action': 'block', 'source': ['192.168.1.0/24'], 'appgroups': [{'id': group, 'name': 'Group one'}], + 'match_all': False, 'managed': True, 'index': 0 } @@ -805,34 +806,82 @@ def test_build_rule_criteria_without_source(e_uci_rules): def test_build_rule_criteria_of_a_rule_matching_nothing(e_uci_rules): - assert dpi.build_rule_criteria(e_uci_rules, {'source': ['192.168.1.1']}) == '' + assert dpi.build_rule_criteria(e_uci_rules, {}) == '' -def test_build_rule_criteria_source_only_when_appgroup_not_required(e_uci_rules): - rule = {'source': ['192.168.1.1', '192.168.1.0/24']} - assert dpi.build_rule_criteria(e_uci_rules, rule, require_appgroup=False) == ( +def test_build_rule_criteria_of_a_match_all_rule(e_uci_rules): + # the wildcard stands alone and carries no terminator + assert dpi.build_rule_criteria(e_uci_rules, {'ns_match_all': '1'}) == '*' + + +def test_build_rule_criteria_of_a_match_all_rule_narrowed_to_a_source(e_uci_rules): + # the wildcard cannot be ANDed with anything: a source expresses the same thing on its own + rule = {'ns_match_all': '1', 'source': ['192.168.1.1', '192.168.1.0/24']} + assert dpi.build_rule_criteria(e_uci_rules, rule) == ( '(local_ip == 192.168.1.1 || local_ip == 192.168.1.0/24);' ) -def test_build_rule_criteria_still_empty_without_source_or_appgroup(e_uci_rules): - assert dpi.build_rule_criteria(e_uci_rules, {}, require_appgroup=False) == '' +def test_build_rule_criteria_source_only_without_the_flag(e_uci_rules): + # a rule migrated before ns_match_all existed carries the same shape and must keep working + rule = {'source': ['192.168.1.1']} + assert dpi.build_rule_criteria(e_uci_rules, rule) == '(local_ip == 192.168.1.1);' -def test_add_rule_without_appgroup_requires_the_flag(e_uci_rules): +def test_add_rule_without_appgroup_requires_match_all(e_uci_rules): with pytest.raises(ValidationError) as err: dpi.add_rule(e_uci_rules, 'No group', True, 'allow', ['192.168.1.1'], []) assert err.value.args[1] == 'appgroups_required' -def test_add_rule_source_only_via_migration_flag(e_uci_rules): - rule_id = dpi.add_rule(e_uci_rules, 'Migrated exception 1', True, 'allow', ['192.168.1.1'], [], - require_appgroup=False) +def test_add_match_all_rule(e_uci_rules): + rule_id = dpi.add_rule(e_uci_rules, 'Allow everything', True, 'allow', [], [], match_all=True) assert e_uci_rules.get('dpi', rule_id, 'ns_managed') == '1' + assert e_uci_rules.get('dpi', rule_id, 'ns_match_all') == '1' + assert e_uci_rules.get('dpi', rule_id, 'appgroup', default=None) is None + assert e_uci_rules.get('dpi', rule_id, 'source', default=None) is None + assert dpi.build_rule_criteria(e_uci_rules, e_uci_rules.get_all('dpi', rule_id)) == '*' + + +def test_add_match_all_rule_narrowed_to_a_source(e_uci_rules): + rule_id = dpi.add_rule(e_uci_rules, 'Allow the office', True, 'allow', ['192.168.1.1'], [], + match_all=True) + assert e_uci_rules.get('dpi', rule_id, 'ns_match_all') == '1' assert e_uci_rules.get('dpi', rule_id, 'source', list=True) == ('192.168.1.1',) assert e_uci_rules.get('dpi', rule_id, 'appgroup', default=None) is None +def test_add_match_all_rule_refuses_an_appgroup(e_uci_rules): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'Contradiction', True, 'allow', [], [group], match_all=True) + assert err.value.args[1] == 'appgroups_not_allowed_with_match_all' + + +def test_list_rules_reports_match_all(e_uci_rules): + group = group_of(e_uci_rules) + dpi.add_rule(e_uci_rules, 'Allow everything', True, 'allow', [], [], match_all=True) + dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + assert [rule['match_all'] for rule in dpi.list_rules(e_uci_rules)] == [True, False] + + +def test_edit_rule_turns_match_all_on(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + dpi.edit_rule(e_uci_rules, rule_id, 'Allow everything', True, 'allow', [], [], match_all=True) + assert e_uci_rules.get('dpi', rule_id, 'ns_match_all') == '1' + assert e_uci_rules.get('dpi', rule_id, 'appgroup', default=None) is None + + +def test_edit_rule_turns_match_all_off(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Allow everything', True, 'allow', [], [], match_all=True) + dpi.edit_rule(e_uci_rules, rule_id, 'Block streaming', True, 'block', [], [group]) + # the flag is removed, not stored as '0' + assert e_uci_rules.get('dpi', rule_id, 'ns_match_all', default=None) is None + assert e_uci_rules.get('dpi', rule_id, 'appgroup', list=True) == (group,) + + # Migration from the schema used before application groups existed def test_freeze_legacy_criteria_from_application_and_device(e_uci: EUci): @@ -974,6 +1023,7 @@ def test_migrate_schema_converts_address_exemption_to_managed_allow_rule(e_uci_l assert exception1['action'] == 'allow' assert exception1['source'] == ['192.168.122.47'] assert exception1['appgroups'] == [] + assert exception1['match_all'] is True assert exception1['enabled'] is True From 6042c551707f7ccc47a1bfb9a2c48e800128a9f7 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Tue, 22 Sep 2026 15:40:09 +0200 Subject: [PATCH 07/17] feat(dpi): update UI source version to specific commit v4 --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 3e42e7b73..ea61467bc 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=a1c0cbc56c85742a8dc1bd5d690063c5ad4f4373 +PKG_SOURCE_VERSION:=9216dced5ae451d7684e348d34019274fc01c20e PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip From 2dddc87a7de56e9e30ff09f9578522653481fa72 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Thu, 24 Sep 2026 12:50:54 +0200 Subject: [PATCH 08/17] feat(dpi): add new API for complete application groups catalog --- packages/ns-api/README.md | 150 +++++++++++------- packages/ns-api/files/ns.dpi | 20 +-- .../src/nethsec/dpi/__init__.py | 126 +++++++++++++++ packages/python3-nethsec/tests/test_dpi.py | 126 +++++++++++++++ 4 files changed, 345 insertions(+), 77 deletions(-) diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 9b5e3a156..c2673e8d3 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -3550,83 +3550,111 @@ Response example: Manage netifyd DPI engine. -### list-loaded-applications +### list-appgroup-catalog -List the applications the netifyd engine has loaded in memory, as returned by `netifyd --dump-apps`: +List everything an application group can be built with: the catalogs crossed with what the netifyd +engine has loaded, grouped by category, done once on the firewall directly. ```bash -api-cli ns.dpi list-loaded-applications +api-cli ns.dpi list-appgroup-catalog ``` -The engine loads only the applications available inside the installed signatures, so the response can be matched -against `list-application-catalog` to find out which applications of the catalog are usable on the machine. - Example response: -```json -{ - "values": [ - { - "id": 133, - "name": "netify.netflix" - }, - { - "id": 10119, - "name": "netify.adobe" - }, - { - "id": 10552, - "name": "netify.tesla" - } - ] -} -``` -If the engine can't be queried, the response is: ```json { - "error": "applications_not_available" + "values": { + "applications": [ + { + "tag": "portal", + "label": "Portal", + "selectable": true, + "items": [ + { + "id": "netify.qq", + "label": "Tencent QQ", + "selectable": true, + "logo": "https://static.netify.ai/logos/q/q//dd/icon.png?v=4" + }, + { + "id": "netify.premium-only", + "label": "Premium Only", + "selectable": false + } + ] + }, + { + "tag": "", + "label": "", + "selectable": false, + "items": [ + { + "id": "netify.brand-new", + "label": "netify.brand-new", + "selectable": true + } + ] + } + ], + "protocols": [ + { + "tag": "file-server", + "label": "File Server", + "selectable": true, + "items": [ + { + "id": "FTP", + "label": "FTP Control", + "selectable": true + } + ] + } + ] + } } ``` -### list-loaded-protocols +**`selectable` on an item** means the engine has the signature loaded, so the item can be matched on +this machine. Unregistering a subscription strips the premium signatures and drops the matchable set +from ~2600 applications to ~200, while the catalog still lists them all: catalog presence is not +matchability. -List the protocols the netifyd engine has loaded in memory, as returned by `netifyd --dump-protos`: +**`id` is only submittable when `selectable` is `true`.** It carries the engine name whenever there is +one, which is what `add-appgroup` stores and what a criteria matches on. For an item the engine has not +loaded there is no engine name, so the catalog tag stands in — and the two differ, `ftp-control` in the +catalog against `FTP` in the engine. Sending the id of a non-selectable item is refused with +`invalid_application` or `invalid_protocol`. -```bash -api-cli ns.dpi list-loaded-protocols -``` +**`selectable` on a group** means the category itself can be a group member, passed as +`application_categories` or `protocol_categories` to `add-appgroup`. Every real category can; the group +with an empty `tag` is the bucket holding whatever the catalog does not categorise, it is not a category +and no criteria can match on it, so it is never selectable as a whole. Its items are selectable +individually like any other. -Like `list-loaded-applications`, the response can be matched against `list-protocol-catalog` to find out which -protocols of the catalog are usable on the machine. The engine can also report protocols that are not part of the -catalog, like the `Unknown` pseudo-protocol with id `0`. +**`label` is English**, the only language the catalog carries, and `logo` is absent when the catalog has +no icon for the entry — always for protocols. A client showing translated category names should +translate by `tag` and keep `label` as the fallback, then sort the groups again: they arrive sorted by +their English label, with the uncategorised group last. -Example response: -```json -{ - "values": [ - { - "id": 116, - "name": "Warcraft3" - }, - { - "id": 121, - "name": "Dropbox" - }, - { - "id": 130, - "name": "HTTP/Connect" - } - ] -} -``` +What is left out, and why: + +- applications the catalog marks `active: false` and protocols it marks deprecated: they are gone from + the product +- the `unknown` protocol: it is the engine's catch-all for what it could not classify +- nothing else — what the engine loaded but the catalog does not list is kept, named after the engine + and left uncategorised, because it is matchable all the same + +Error response: -If the engine can't be queried, the response is: ```json -{ - "error": "protocols_not_available" -} +{"error": "catalog_not_available"} ``` +Returned whenever any source is missing: the engine is not running, or a catalog has not been downloaded +yet. The answer is all-or-nothing on purpose — a partial one would quietly offer a smaller product than +the machine actually has. The catalogs are refreshed nightly by `dpi-data-update` into `/etc/netifyd`, so +a machine that has never reached `distfeed.nethesis.it` has none. + ### list-appgroups List the application groups, ordered by name: @@ -3686,10 +3714,10 @@ api-cli ns.dpi add-appgroup --data '{ - `name`: mandatory, up to 64 characters, unique among the groups regardless of case - the four member lists are all optional, but the group must hold at least one member overall -- applications and protocols are named as the **DPI engine** reports them, i.e. the `name` field of - `list-loaded-applications` / `list-loaded-protocols`, **not** the `tag` of the catalogs. For - applications the two coincide (`netify.amazon`); for protocols they don't (`HTTP/Connect` in the - engine, `http-connect` in the catalog) and only the engine name can be matched +- applications and protocols are named as the **DPI engine** reports them, which is the `id` of a + selectable item of `list-appgroup-catalog`, **not** the `tag` of the raw catalogs. For applications + the two coincide (`netify.amazon`); for protocols they don't (`HTTP/Connect` in the engine, + `http-connect` in the catalog) and only the engine name can be matched - categories are named by tag (`cybersecurity`, `games`), which is the same in the engine and in the catalogs. Application and protocol categories are distinct vocabularies diff --git a/packages/ns-api/files/ns.dpi b/packages/ns-api/files/ns.dpi index db9dbaca9..3b2ec9e39 100755 --- a/packages/ns-api/files/ns.dpi +++ b/packages/ns-api/files/ns.dpi @@ -51,9 +51,8 @@ if cmd == 'list': 'order': ['ns_1234', 'ns_5678'] }, 'list-application-catalog': {}, - 'list-loaded-applications': {}, 'list-protocol-catalog': {}, - 'list-loaded-protocols': {}, + 'list-appgroup-catalog': {}, 'list-appgroups': { 'search': 'str', 'limit': 32, @@ -89,14 +88,6 @@ elif cmd == 'call': print(json.dumps({'values': content})) except Exception: print(json.dumps({})) - elif action == 'list-loaded-applications': - try: - applications = dpi.load_applications() - except Exception: - print(json.dumps(utils.generic_error('applications_not_available'))) - else: - print(json.dumps({'values': [{'id': app_id, 'name': app_name} - for app_id, app_name in applications.items()]})) elif action == 'list-protocol-catalog': try: with open('/etc/netifyd/netify-protocol-catalog.json', 'r') as f: @@ -104,14 +95,11 @@ elif cmd == 'call': print(json.dumps({'values': content})) except Exception: print(json.dumps({})) - elif action == 'list-loaded-protocols': + elif action == 'list-appgroup-catalog': try: - protocols = dpi.load_protocols() + print(json.dumps({'values': dpi.list_appgroup_catalog()})) except Exception: - print(json.dumps(utils.generic_error('protocols_not_available'))) - else: - print(json.dumps({'values': [{'id': proto_id, 'name': proto_name} - for proto_id, proto_name in protocols.items()]})) + print(json.dumps(utils.generic_error('catalog_not_available'))) elif action == 'list-rules': print(json.dumps({'values': dpi.list_rules(e_uci)})) elif action == 'add-rule': diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py index ae1b14d31..3c0c36cd0 100644 --- a/packages/python3-nethsec/src/nethsec/dpi/__init__.py +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -201,6 +201,132 @@ def __appgroup_vocabulary(kind: str) -> set[str] | None: # an unknown value never matches, so a missing vocabulary is a reason to skip the check, not to refuse return vocabulary if vocabulary else None +APPGROUP_CATALOG_FILES = { + 'applications': 'netify-application-catalog.json', + 'protocols': 'netify-protocol-catalog.json' +} + +UNCATEGORIZED_TAG = '' + +def __load_catalog(filename: str) -> list[dict]: + """ + Reads a downloaded catalog, one of the `/etc/netifyd/netify-*-catalog.json` files. + + Args: + - filename: name of the catalog file inside the netifyd data directory + + Returns: + list of catalog entries + """ + with open(f'{NETIFYD_DATA_DIR}/{filename}', 'r') as file: + return json.load(file) + + +def __catalog_entries(catalog: list[dict], loaded: dict[int, str], kind: str) -> list[dict]: + """ + Cross one catalog with what the engine has loaded, into flat entries carrying their category. + + Three rules the engine and the catalog do not state on their own: + + - an application the catalog marks `active: false`, and a protocol it marks deprecated, are gone + from the product and must not be offered + - a protocol labelled "unknown" is the engine's catch-all for what it could not classify, not + something a group can be built on + - what the engine loaded but the catalog does not list is still matchable, so it is kept, named + after the engine and left uncategorised + + The id is what a group stores, and it is the engine name whenever there is one: the catalog tag is + only a fallback for an entry that cannot be matched anyway, so it is never a value a client submits. + """ + entries = [] + for entry in catalog: + if kind == 'applications' and entry.get('active') is False: + continue + if kind == 'protocols' and entry.get('deprecated'): + continue + label = entry.get('label', '') + if kind == 'protocols' and label.lower() == 'unknown': + continue + engine_name = loaded.get(entry.get('id')) + category = entry.get('category') or {} + entries.append({ + 'id': engine_name or entry.get('tag', ''), + 'label': label, + 'selectable': engine_name is not None, + 'logo': entry.get('icon'), + 'category_tag': category.get('tag', UNCATEGORIZED_TAG), + 'category_label': category.get('label', '') + }) + + known = {entry.get('id') for entry in catalog} + for entry_id, name in loaded.items(): + if entry_id in known: + continue + if kind == 'protocols' and name.lower() == 'unknown': + continue + entries.append({ + 'id': name, + 'label': name, + 'selectable': True, + 'logo': None, + 'category_tag': UNCATEGORIZED_TAG, + 'category_label': '', + }) + + return entries + + +def __catalog_groups(entries: list[dict]) -> list[dict]: + """ + Group flat catalog entries by category. + + A group is selectable when the category itself can be a group member, which every real category can + be and the uncategorised bucket never can. Groups and items are sorted by label, with the + uncategorised bucket last; a client translating the category labels has to sort them again. + """ + groups = dict[str, dict]() + for entry in entries: + tag = entry['category_tag'] + group = groups.get(tag) + if group is None: + group = groups[tag] = { + 'tag': tag, + 'label': entry['category_label'], + 'selectable': tag != UNCATEGORIZED_TAG, + 'items': [] + } + item = {'id': entry['id'], 'label': entry['label'], 'selectable': entry['selectable']} + if entry['logo']: + item['logo'] = entry['logo'] + group['items'].append(item) + + for group in groups.values(): + group['items'].sort(key=lambda item: item['label'].lower()) + + return sorted(groups.values(), key=lambda group: (group['tag'] == UNCATEGORIZED_TAG, + group['label'].lower())) + + +def list_appgroup_catalog() -> dict[str, list[dict]]: + """ + List every application and protocol a group can be built with, grouped by category. + + Every source has to answer: the engine says what is matchable, the catalogs say how it reads, and a + partial answer would silently offer a smaller product than the machine has. So this raises instead + of returning what it managed to collect. + + Returns: + dict with the keys "applications" and "protocols", each a list of category groups + + Raises: + - Exception: if the engine cannot be queried or a catalog is missing + """ + loaded = {'applications': load_applications(), 'protocols': load_protocols()} + return { + kind: __catalog_groups(__catalog_entries(__load_catalog(filename), loaded[kind], kind)) + for kind, filename in APPGROUP_CATALOG_FILES.items() + } + def __validate_appgroup_members(kind: str, values: list[str]) -> list[str]: """ diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py index c4e35cf04..e71a99f49 100644 --- a/packages/python3-nethsec/tests/test_dpi.py +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -520,6 +520,132 @@ def test_expand_appgroups_without_groups(e_uci_appgroups): assert dpi.expand_appgroups(e_uci_appgroups, []) == '' +# appgroup catalog + +BUSINESS = {'id': 5, 'tag': 'business', 'label': 'Business'} +PORTAL = {'id': 19, 'tag': 'portal', 'label': 'Portal'} +FILE_SERVER = {'id': 4, 'tag': 'file-server', 'label': 'File Server'} + +application_catalog = [ + {'id': 133, 'tag': 'netify.netflix', 'label': 'Netflix', 'category': PORTAL, 'active': True, + 'icon': 'https://static.netify.ai/logos/n/netflix/icon.png'}, + {'id': 10552, 'tag': 'netify.tesla', 'label': 'Tesla', 'category': BUSINESS}, + # not loaded by the engine: listed, named after the catalog, not selectable + {'id': 90001, 'tag': 'netify.premium-only', 'label': 'Premium Only', 'category': BUSINESS}, + # retired from the product + {'id': 90002, 'tag': 'netify.retired', 'label': 'Retired', 'category': BUSINESS, 'active': False}, + # no category at all + {'id': 10119, 'tag': 'netify.linkedin', 'label': 'LinkedIn'} +] + +protocol_catalog = [ + {'id': 1, 'tag': 'ftp-control', 'label': 'FTP Control', 'category': FILE_SERVER, 'deprecated': ''}, + {'id': 130, 'tag': 'http-connect', 'label': 'HTTP/Connect', 'category': FILE_SERVER}, + {'id': 90003, 'tag': 'gone', 'label': 'Gone', 'category': FILE_SERVER, 'deprecated': '5.0'}, + {'id': 90004, 'tag': 'unknown', 'label': 'Unknown', 'category': FILE_SERVER} +] + + +@pytest.fixture +def mock_catalogs(mocker): + """Both catalogs are on disk and the engine has loaded part of them.""" + mocker.patch('nethsec.dpi.load_applications', return_value={ + 133: 'netify.netflix', 10552: 'netify.tesla', 10119: 'netify.linkedin', + # loaded but absent from the catalog + 99999: 'netify.brand-new' + }) + mocker.patch('nethsec.dpi.load_protocols', return_value={ + 1: 'FTP', 130: 'HTTP/Connect', 99998: 'Unknown' + }) + mocker.patch('nethsec.dpi.__load_catalog', side_effect=lambda filename: ( + application_catalog if 'application' in filename else protocol_catalog + )) + + +def group_by_tag(groups, tag): + return next(group for group in groups if group['tag'] == tag) + + +def test_list_appgroup_catalog_groups_by_category(mock_catalogs): + catalog = dpi.list_appgroup_catalog() + assert [group['tag'] for group in catalog['applications']] == ['business', 'portal', ''] + assert group_by_tag(catalog['applications'], 'portal')['label'] == 'Portal' + + +def test_list_appgroup_catalog_marks_every_real_category_selectable(mock_catalogs): + catalog = dpi.list_appgroup_catalog() + assert group_by_tag(catalog['applications'], 'business')['selectable'] is True + # the uncategorised bucket is not a category: no criteria can match on it + assert group_by_tag(catalog['applications'], '')['selectable'] is False + + +def test_list_appgroup_catalog_item_of_a_loaded_application(mock_catalogs): + items = group_by_tag(dpi.list_appgroup_catalog()['applications'], 'portal')['items'] + assert items == [{'id': 'netify.netflix', 'label': 'Netflix', 'selectable': True, + 'logo': 'https://static.netify.ai/logos/n/netflix/icon.png'}] + + +def test_list_appgroup_catalog_item_the_engine_did_not_load(mock_catalogs): + items = group_by_tag(dpi.list_appgroup_catalog()['applications'], 'business')['items'] + premium = next(item for item in items if item['label'] == 'Premium Only') + # no engine name to store, so the catalog tag stands in and the item cannot be picked + assert premium == {'id': 'netify.premium-only', 'label': 'Premium Only', 'selectable': False} + + +def test_list_appgroup_catalog_hides_an_inactive_application(mock_catalogs): + catalog = dpi.list_appgroup_catalog() + labels = [item['label'] for group in catalog['applications'] for item in group['items']] + assert 'Retired' not in labels + + +def test_list_appgroup_catalog_hides_a_deprecated_protocol(mock_catalogs): + catalog = dpi.list_appgroup_catalog() + labels = [item['label'] for group in catalog['protocols'] for item in group['items']] + assert 'Gone' not in labels + + +def test_list_appgroup_catalog_hides_the_unknown_protocol(mock_catalogs): + catalog = dpi.list_appgroup_catalog() + ids = [item['id'] for group in catalog['protocols'] for item in group['items']] + # dropped both from the catalog and from what the engine loaded + assert 'unknown' not in ids and 'Unknown' not in ids + + +def test_list_appgroup_catalog_keeps_what_the_catalog_does_not_list(mock_catalogs): + items = group_by_tag(dpi.list_appgroup_catalog()['applications'], '')['items'] + assert {'id': 'netify.brand-new', 'label': 'netify.brand-new', 'selectable': True} in items + + +def test_list_appgroup_catalog_uses_the_engine_name_as_id(mock_catalogs): + # the catalog calls it 'ftp-control', the engine calls it 'FTP': a group stores the engine name + items = group_by_tag(dpi.list_appgroup_catalog()['protocols'], 'file-server')['items'] + assert next(item for item in items if item['label'] == 'FTP Control')['id'] == 'FTP' + + +def test_list_appgroup_catalog_omits_the_logo_when_there_is_none(mock_catalogs): + items = group_by_tag(dpi.list_appgroup_catalog()['applications'], 'business')['items'] + assert 'logo' not in next(item for item in items if item['label'] == 'Tesla') + + +def test_list_appgroup_catalog_sorts_items_by_label(mock_catalogs): + items = group_by_tag(dpi.list_appgroup_catalog()['applications'], 'business')['items'] + assert [item['label'] for item in items] == ['Premium Only', 'Tesla'] + + +def test_list_appgroup_catalog_raises_when_the_engine_is_down(mocker): + mocker.patch('nethsec.dpi.load_applications', side_effect=Exception('netifyd is not running')) + with pytest.raises(Exception): + dpi.list_appgroup_catalog() + + +def test_list_appgroup_catalog_raises_when_a_catalog_is_missing(mocker): + mocker.patch('nethsec.dpi.load_applications', return_value={}) + mocker.patch('nethsec.dpi.load_protocols', return_value={}) + mocker.patch('nethsec.dpi.__load_catalog', side_effect=FileNotFoundError) + with pytest.raises(FileNotFoundError): + dpi.list_appgroup_catalog() + + # rules @pytest.fixture From dea960b10a8c55d88deaa2766f58f408823ed2ec Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Thu, 24 Sep 2026 12:51:45 +0200 Subject: [PATCH 09/17] feat(dpi): update UI source version to specific commit v5 --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index ea61467bc..17abf6633 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=9216dced5ae451d7684e348d34019274fc01c20e +PKG_SOURCE_VERSION:=156c59182a614c1ed8ecd3f5c9463d65bfae0bf9 PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip From 834fd959ae3d5d5f4e591d08d44e05789f0aca23 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Fri, 25 Sep 2026 12:29:33 +0200 Subject: [PATCH 10/17] feat(dpi): update UI source version to specific commit v6 --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 17abf6633..90b315335 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=156c59182a614c1ed8ecd3f5c9463d65bfae0bf9 +PKG_SOURCE_VERSION:=588121f04ffbd1c7df7f9959f005578afc75a415 PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip From 4e5da91922a2d40168078791ba2b707c0f92e311 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Thu, 1 Oct 2026 12:18:14 +0200 Subject: [PATCH 11/17] feat(dpi): handle firewall exemptions with hotplug --- .../netifyd/references/nethsecurity-wiring.md | 5 +++- packages/ns-dpi/Makefile | 3 +++ packages/ns-dpi/README.md | 7 +++++- packages/ns-dpi/files/20_dpi | 2 +- .../ns-dpi/files/22_dpi_firewall_exemption | 14 +++++++++++ packages/ns-dpi/files/dpi-config | 2 +- packages/ns-dpi/files/dpi-exemption.hotplug | 25 +++++++++++++++++++ 7 files changed, 54 insertions(+), 4 deletions(-) create mode 100755 packages/ns-dpi/files/22_dpi_firewall_exemption create mode 100755 packages/ns-dpi/files/dpi-exemption.hotplug diff --git a/.agents/skills/netifyd/references/nethsecurity-wiring.md b/.agents/skills/netifyd/references/nethsecurity-wiring.md index 1876e961f..31b9b8c1c 100644 --- a/.agents/skills/netifyd/references/nethsecurity-wiring.md +++ b/.agents/skills/netifyd/references/nethsecurity-wiring.md @@ -99,7 +99,10 @@ Current generator behaviour worth knowing before touching it: so a rule carrying both its own `criteria` and a `device` gets the term whether or not it already has one. - A hidden `analyzed` action (`detection_guessed || detection_complete;`) is appended last, labelling every analysed flow. -- `dpi.config.firewall_exemption` optionally pre-fills `exemptions` with every firewall interface IP; +- `dpi.config.firewall_exemption` (default `1`, forced to `1` on every upgrade by `22_dpi_firewall_exemption`) + pre-fills `exemptions` with every firewall interface IP; `/etc/hotplug.d/iface/90-dpi-exemption` re-runs + `dpi-config` on `ifup` / address `ifupdate` and does `netifyd reload` only if the JSON changed and the + agent is running (so HA backup nodes are left alone); `exemption` sections add more, and an exemption whose criteria is an object ID is expanded to its IPs. - `dpi.config.log_blocked` drives the nft log rule in `dpi-nft`, not any netify logging target. diff --git a/packages/ns-dpi/Makefile b/packages/ns-dpi/Makefile index 17af07c6b..0b721f77a 100644 --- a/packages/ns-dpi/Makefile +++ b/packages/ns-dpi/Makefile @@ -66,6 +66,7 @@ define Package/ns-dpi/install $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/unregister $(INSTALL_DIR) $(1)/etc/xtables + $(INSTALL_DIR) $(1)/etc/hotplug.d/iface $(INSTALL_CONF) ./files/connlabel.conf $(1)/etc/connlabel.conf $(LN) /etc/connlabel.conf $(1)/etc/xtables/connlabel.conf $(INSTALL_BIN) ./files/dpi.init $(1)/etc/init.d/dpi @@ -81,10 +82,12 @@ define Package/ns-dpi/install $(INSTALL_BIN) ./files/dpi-migrate.py $(1)/usr/sbin/dpi-migrate $(INSTALL_CONF) ./files/20_dpi $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/21_dpi_migrate $(1)/etc/uci-defaults/21_dpi_migrate + $(INSTALL_BIN) ./files/22_dpi_firewall_exemption $(1)/etc/uci-defaults/22_dpi_firewall_exemption $(INSTALL_BIN) ./files/99-dpi-license-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-license-update-cron $(INSTALL_BIN) ./files/99-dpi-data-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-data-update-cron $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/register $(INSTALL_BIN) ./files/70dpi $(1)/usr/share/ns-plug/hooks/unregister/ + $(INSTALL_BIN) ./files/dpi-exemption.hotplug $(1)/etc/hotplug.d/iface/90-dpi-exemption endef $(eval $(call BuildPackage,ns-dpi)) diff --git a/packages/ns-dpi/README.md b/packages/ns-dpi/README.md index ff267ad6b..7467508fe 100644 --- a/packages/ns-dpi/README.md +++ b/packages/ns-dpi/README.md @@ -21,7 +21,12 @@ Global options: - `enabled`: can be `0` or `1`, if set to `1` enable the service - `log_blocked`: can be `0` or `1`, if set to `1` blocked connections will be logged inside `/var/log/messages` - `firewall_exemption`: can be `0` or `1`, if set to `1` all firewall IP addresses will be - added to global exemption list and will not match DPI rules + added to global exemption list and will not match DPI rules; default is `1`, so DPI rules can not block + access to the firewall itself (traffic forwarded between hosts is still filtered). + The option is forced to `1` on every image upgrade, so a `0` set by hand does not survive an upgrade. + Addresses changing at runtime (DHCP, PPPoE) are tracked by `/etc/hotplug.d/iface/90-dpi-exemption`: on + `ifup` or an address `ifupdate` it regenerates the flow actions config and, only if it changed and netifyd + is running, reloads netifyd - `popular_filters`: list of filters that will be returned to from `api-cli ns.dpi list-popular` call. - `ns_exclude`: list of network interface exclusions in Netifyd that will be returned by `uci show netifyd.@netifyd[0].ns_exclude` diff --git a/packages/ns-dpi/files/20_dpi b/packages/ns-dpi/files/20_dpi index da9094280..e0677772a 100644 --- a/packages/ns-dpi/files/20_dpi +++ b/packages/ns-dpi/files/20_dpi @@ -5,7 +5,7 @@ if ! uci -q get dpi.config > /dev/null; then config main 'config' option log_blocked '0' option enabled '0' - option firewall_exemption '0' + option firewall_exemption '1' EOI fi diff --git a/packages/ns-dpi/files/22_dpi_firewall_exemption b/packages/ns-dpi/files/22_dpi_firewall_exemption new file mode 100755 index 000000000..15463c920 --- /dev/null +++ b/packages/ns-dpi/files/22_dpi_firewall_exemption @@ -0,0 +1,14 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Enable firewall_exemption on systems installed before it became the default, so DPI rules can not lock +# the administrator out of the unit. + +if [ "$(uci -q get dpi.config.firewall_exemption)" != "1" ]; then + uci -q set dpi.config.firewall_exemption='1' + uci commit dpi +fi diff --git a/packages/ns-dpi/files/dpi-config b/packages/ns-dpi/files/dpi-config index f5bc79e6a..ee9df2268 100755 --- a/packages/ns-dpi/files/dpi-config +++ b/packages/ns-dpi/files/dpi-config @@ -15,7 +15,7 @@ from nethsec import utils, objects, dpi def get_interface_ips(interface): ret = list() try: - proc = subprocess.run(['/sbin/ifstatus', section], check=True, capture_output=True) + proc = subprocess.run(['/sbin/ifstatus', interface], check=True, capture_output=True) info = json.loads(proc.stdout) except: return ret diff --git a/packages/ns-dpi/files/dpi-exemption.hotplug b/packages/ns-dpi/files/dpi-exemption.hotplug new file mode 100755 index 000000000..a1a60a86f --- /dev/null +++ b/packages/ns-dpi/files/dpi-exemption.hotplug @@ -0,0 +1,25 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Keep the firewall_exemption list in sync with interface addresses that change at runtime (e.g. DHCP or +# PPPoE on WAN). Only netifyd is reloaded, never restarted, and only when the generated config changed. + +[ "$ACTION" = "ifup" ] || { [ "$ACTION" = "ifupdate" ] && [ "$IFUPDATE_ADDRESSES" = "1" ]; } || exit 0 +[ "$(uci -q get dpi.config.firewall_exemption)" = "1" ] || exit 0 + +cfg_file=/etc/netifyd/netify-proc-flow-actions.json + +lock /var/lock/dpi-exemption.lock +before=$(md5sum "$cfg_file" 2>/dev/null) +/usr/sbin/dpi-config +after=$(md5sum "$cfg_file" 2>/dev/null) +lock -u /var/lock/dpi-exemption.lock + +# a stopped agent (e.g. HA backup node) reads the updated file when it starts +if [ "$before" != "$after" ] && /etc/init.d/netifyd running; then + /etc/init.d/netifyd reload +fi From 670348276cf1f8dceb53ad0b30ff00992a1e1efd Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Thu, 1 Oct 2026 17:44:01 +0200 Subject: [PATCH 12/17] feat(dpi): streamline json handling and update configuration management --- packages/ns-api/files/ns.dpi | 22 +- packages/ns-dpi/Makefile | 4 +- packages/ns-dpi/README.md | 17 +- packages/ns-dpi/files/20_dpi | 1 - packages/ns-dpi/files/21_dpi_migrate | 2 +- packages/ns-dpi/files/23_dpi_drop_enabled | 14 ++ .../src/nethsec/dpi/__init__.py | 218 ++++++------------ .../src/nethsec/inventory/__init__.py | 2 +- packages/python3-nethsec/tests/test_dpi.py | 105 +++++---- .../python3-nethsec/tests/test_inventory.py | 6 +- 10 files changed, 166 insertions(+), 225 deletions(-) create mode 100755 packages/ns-dpi/files/23_dpi_drop_enabled diff --git a/packages/ns-api/files/ns.dpi b/packages/ns-api/files/ns.dpi index 3b2ec9e39..aa6a6d002 100755 --- a/packages/ns-api/files/ns.dpi +++ b/packages/ns-api/files/ns.dpi @@ -103,55 +103,55 @@ elif cmd == 'call': elif action == 'list-rules': print(json.dumps({'values': dpi.list_rules(e_uci)})) elif action == 'add-rule': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) config_name = dpi.add_rule(e_uci, data['name'], data['enabled'], data['action'], data.get('source', []), data.get('appgroups', []), data.get('position', 'bottom'), data.get('match_all', False)) print(json.dumps({'id': config_name})) elif action == 'edit-rule': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) config_name = dpi.edit_rule(e_uci, data['id'], data['name'], data['enabled'], data['action'], data.get('source', []), data.get('appgroups', []), data.get('match_all', False)) print(json.dumps({'id': config_name})) elif action == 'delete-rule': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) dpi.delete_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) elif action == 'rename-rule': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) dpi.rename_rule(e_uci, data['id'], data['name']) print(json.dumps({'message': 'success'})) elif action == 'enable-rule': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) dpi.enable_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) elif action == 'disable-rule': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) dpi.disable_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) elif action == 'order-rules': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) print(json.dumps({'values': dpi.order_rules(e_uci, data['order'])})) elif action == 'list-appgroups': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) result = dpi.list_appgroups(e_uci, data.get('search', None), data.get('limit', None), data.get('page', 1)) print(json.dumps({'values': result})) elif action == 'add-appgroup': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) config_name = dpi.add_appgroup(e_uci, data['name'], data.get('applications', []), data.get('application_categories', []), data.get('protocols', []), data.get('protocol_categories', [])) print(json.dumps({'id': config_name})) elif action == 'edit-appgroup': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) config_name = dpi.edit_appgroup(e_uci, data['id'], data['name'], data.get('applications', []), data.get('application_categories', []), data.get('protocols', []), data.get('protocol_categories', [])) print(json.dumps({'id': config_name})) elif action == 'delete-appgroup': - data = json.JSONDecoder().decode(sys.stdin.read()) + data = json.load(sys.stdin) dpi.delete_appgroup(e_uci, data['id']) print(json.dumps({'message': 'success'})) except KeyError as e: diff --git a/packages/ns-dpi/Makefile b/packages/ns-dpi/Makefile index 0b721f77a..06e8e5f6e 100644 --- a/packages/ns-dpi/Makefile +++ b/packages/ns-dpi/Makefile @@ -62,6 +62,7 @@ endef define Package/ns-dpi/install $(INSTALL_DIR) $(1)/usr/sbin + $(INSTALL_DIR) $(1)/usr/libexec/ns-dpi $(INSTALL_DIR) $(1)/etc/init.d $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/unregister @@ -79,10 +80,11 @@ define Package/ns-dpi/install $(INSTALL_BIN) ./files/dpi-license-update.py $(1)/usr/sbin/dpi-license-update $(INSTALL_BIN) ./files/dpi-data-update.py $(1)/usr/sbin/dpi-data-update $(INSTALL_BIN) ./files/dpi-update.py $(1)/usr/sbin/dpi-update - $(INSTALL_BIN) ./files/dpi-migrate.py $(1)/usr/sbin/dpi-migrate + $(INSTALL_BIN) ./files/dpi-migrate.py $(1)/usr/libexec/ns-dpi/dpi-migrate $(INSTALL_CONF) ./files/20_dpi $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/21_dpi_migrate $(1)/etc/uci-defaults/21_dpi_migrate $(INSTALL_BIN) ./files/22_dpi_firewall_exemption $(1)/etc/uci-defaults/22_dpi_firewall_exemption + $(INSTALL_BIN) ./files/23_dpi_drop_enabled $(1)/etc/uci-defaults/23_dpi_drop_enabled $(INSTALL_BIN) ./files/99-dpi-license-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-license-update-cron $(INSTALL_BIN) ./files/99-dpi-data-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-data-update-cron $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/register diff --git a/packages/ns-dpi/README.md b/packages/ns-dpi/README.md index 7467508fe..c4027a014 100644 --- a/packages/ns-dpi/README.md +++ b/packages/ns-dpi/README.md @@ -6,19 +6,15 @@ How it works: - Netify flow actions plugin adds a label to matching connections - nft rules can block or change priority (`dscp`) to connections with labels -To enable traffic processing: -- configure `dpi` UCI database (see below for an example) -- enable DPI service: - ``` - uci set dpi.config.enabled=1 - uci commit dpi - /etc/init.d/dpi restart - service netifyd reload - ``` +netifyd runs all the time: traffic is filtered as soon as at least one rule is enabled. +To apply a change made by hand to the `dpi` UCI database (see below for an example): +``` +uci commit dpi +/etc/init.d/dpi reload +``` Global options: -- `enabled`: can be `0` or `1`, if set to `1` enable the service - `log_blocked`: can be `0` or `1`, if set to `1` blocked connections will be logged inside `/var/log/messages` - `firewall_exemption`: can be `0` or `1`, if set to `1` all firewall IP addresses will be added to global exemption list and will not match DPI rules; default is `1`, so DPI rules can not block @@ -61,7 +57,6 @@ Example of `/etc/config/dpi`: ``` config main 'config' option log_blocked '1' - option enabled '1' option firewall_exemption '1' list popular_filters 'netify.netflix' list popular_filters 'netify.telegram' diff --git a/packages/ns-dpi/files/20_dpi b/packages/ns-dpi/files/20_dpi index e0677772a..662d78a76 100644 --- a/packages/ns-dpi/files/20_dpi +++ b/packages/ns-dpi/files/20_dpi @@ -4,7 +4,6 @@ if ! uci -q get dpi.config > /dev/null; then uci -q import dpi << EOI config main 'config' option log_blocked '0' - option enabled '0' option firewall_exemption '1' EOI fi diff --git a/packages/ns-dpi/files/21_dpi_migrate b/packages/ns-dpi/files/21_dpi_migrate index 4521e6801..839b49e22 100755 --- a/packages/ns-dpi/files/21_dpi_migrate +++ b/packages/ns-dpi/files/21_dpi_migrate @@ -10,4 +10,4 @@ set -e -/usr/sbin/dpi-migrate +/usr/libexec/ns-dpi/dpi-migrate diff --git a/packages/ns-dpi/files/23_dpi_drop_enabled b/packages/ns-dpi/files/23_dpi_drop_enabled new file mode 100755 index 000000000..f1e002ab9 --- /dev/null +++ b/packages/ns-dpi/files/23_dpi_drop_enabled @@ -0,0 +1,14 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Remove the global enabled flag from systems installed before netifyd ran all the time: it no longer +# switches anything, DPI filters traffic as soon as one rule is enabled. + +if uci -q get dpi.config.enabled > /dev/null; then + uci -q delete dpi.config.enabled + uci commit dpi +fi diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py index 3c0c36cd0..e6b16b6b4 100644 --- a/packages/python3-nethsec/src/nethsec/dpi/__init__.py +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -66,30 +66,6 @@ def load_protocols() -> dict[int, str]: return protocols -def __apply(e_uci: EUci): - """ - Commit the dpi config and reload the dpi service. - - Used by the schema migration only: it runs at boot with nobody around to confirm the pending uci - changes, so it must take effect on its own. Rule and appgroup CRUD go through the normal pending - uci changes instead, applied later by ns.commit like every other section. - """ - e_uci.commit('dpi') - subprocess.run(["/etc/init.d/dpi", "reload"], check=True) - - -def __toggle_engine(e_uci: EUci): - count_enabled = 0 - for section in e_uci.get_all('dpi'): - if e_uci.get('dpi', section, default="") == "rule" and e_uci.get('dpi', section, 'enabled', default="0") == "1": - count_enabled = count_enabled + 1 - - if count_enabled > 0: - e_uci.set('dpi', 'config', 'enabled', '1') - else: - e_uci.set('dpi', 'config', 'enabled', '0') - - # Application groups: a NethSecurity abstraction, the netifyd plugin has no group primitive. # A group is a named set of match members, expanded into an inline expression at generation time. @@ -145,18 +121,18 @@ def load_categories() -> dict[str, dict[int, str]]: return categories -def __load_catalog_tags(filename: str) -> set[str]: +def __load_catalog(filename: str) -> list[dict]: """ - Reads the tags of a downloaded catalog, one of the `/etc/netifyd/netify-*.json` files. + Reads a downloaded catalog, one of the `/etc/netifyd/netify-*.json` files. Args: - filename: name of the catalog file inside the netifyd data directory Returns: - set of the tags the catalog lists + list of catalog entries """ with open(f'{NETIFYD_DATA_DIR}/{filename}', 'r') as file: - return {entry['tag'] for entry in json.load(file) if entry.get('tag')} + return json.load(file) def __appgroup_vocabulary(kind: str) -> set[str] | None: @@ -175,12 +151,12 @@ def __appgroup_vocabulary(kind: str) -> set[str] | None: vocabulary = set[str]() try: if kind == 'app': - vocabulary |= {value.lower() for value in load_applications().values()} + vocabulary.update({value.lower() for value in load_applications().values()}) elif kind == 'proto': - vocabulary |= {value.lower() for value in load_protocols().values()} + vocabulary.update({value.lower() for value in load_protocols().values()}) else: category_type = 'application' if kind == 'app_category' else 'protocol' - vocabulary |= {value.lower() for value in load_categories().get(category_type, {}).values()} + vocabulary.update({value.lower() for value in load_categories().get(category_type, {}).values()}) except Exception: # the engine can't be queried, fall back to the downloaded catalogs below pass @@ -194,7 +170,7 @@ def __appgroup_vocabulary(kind: str) -> set[str] | None: 'proto_category': 'netify-protocol-categories.json' } try: - vocabulary |= {value.lower() for value in __load_catalog_tags(catalogs[kind])} + vocabulary.update({entry['tag'].lower() for entry in __load_catalog(catalogs[kind]) if entry.get('tag')}) except Exception: pass @@ -208,104 +184,6 @@ def __appgroup_vocabulary(kind: str) -> set[str] | None: UNCATEGORIZED_TAG = '' -def __load_catalog(filename: str) -> list[dict]: - """ - Reads a downloaded catalog, one of the `/etc/netifyd/netify-*-catalog.json` files. - - Args: - - filename: name of the catalog file inside the netifyd data directory - - Returns: - list of catalog entries - """ - with open(f'{NETIFYD_DATA_DIR}/{filename}', 'r') as file: - return json.load(file) - - -def __catalog_entries(catalog: list[dict], loaded: dict[int, str], kind: str) -> list[dict]: - """ - Cross one catalog with what the engine has loaded, into flat entries carrying their category. - - Three rules the engine and the catalog do not state on their own: - - - an application the catalog marks `active: false`, and a protocol it marks deprecated, are gone - from the product and must not be offered - - a protocol labelled "unknown" is the engine's catch-all for what it could not classify, not - something a group can be built on - - what the engine loaded but the catalog does not list is still matchable, so it is kept, named - after the engine and left uncategorised - - The id is what a group stores, and it is the engine name whenever there is one: the catalog tag is - only a fallback for an entry that cannot be matched anyway, so it is never a value a client submits. - """ - entries = [] - for entry in catalog: - if kind == 'applications' and entry.get('active') is False: - continue - if kind == 'protocols' and entry.get('deprecated'): - continue - label = entry.get('label', '') - if kind == 'protocols' and label.lower() == 'unknown': - continue - engine_name = loaded.get(entry.get('id')) - category = entry.get('category') or {} - entries.append({ - 'id': engine_name or entry.get('tag', ''), - 'label': label, - 'selectable': engine_name is not None, - 'logo': entry.get('icon'), - 'category_tag': category.get('tag', UNCATEGORIZED_TAG), - 'category_label': category.get('label', '') - }) - - known = {entry.get('id') for entry in catalog} - for entry_id, name in loaded.items(): - if entry_id in known: - continue - if kind == 'protocols' and name.lower() == 'unknown': - continue - entries.append({ - 'id': name, - 'label': name, - 'selectable': True, - 'logo': None, - 'category_tag': UNCATEGORIZED_TAG, - 'category_label': '', - }) - - return entries - - -def __catalog_groups(entries: list[dict]) -> list[dict]: - """ - Group flat catalog entries by category. - - A group is selectable when the category itself can be a group member, which every real category can - be and the uncategorised bucket never can. Groups and items are sorted by label, with the - uncategorised bucket last; a client translating the category labels has to sort them again. - """ - groups = dict[str, dict]() - for entry in entries: - tag = entry['category_tag'] - group = groups.get(tag) - if group is None: - group = groups[tag] = { - 'tag': tag, - 'label': entry['category_label'], - 'selectable': tag != UNCATEGORIZED_TAG, - 'items': [] - } - item = {'id': entry['id'], 'label': entry['label'], 'selectable': entry['selectable']} - if entry['logo']: - item['logo'] = entry['logo'] - group['items'].append(item) - - for group in groups.values(): - group['items'].sort(key=lambda item: item['label'].lower()) - - return sorted(groups.values(), key=lambda group: (group['tag'] == UNCATEGORIZED_TAG, - group['label'].lower())) - def list_appgroup_catalog() -> dict[str, list[dict]]: """ @@ -315,6 +193,12 @@ def list_appgroup_catalog() -> dict[str, list[dict]]: partial answer would silently offer a smaller product than the machine has. So this raises instead of returning what it managed to collect. + The id of an item is what a group stores, and it is the engine name whenever there is one: the + catalog tag is only a fallback for an entry that cannot be matched anyway, so it is never a value a + client submits. A group is selectable when the category itself can be a group member, which every + real category can be and the uncategorised bucket never can. Groups and items are sorted by label, + with the uncategorised bucket last; a client translating the category labels has to sort them again. + Returns: dict with the keys "applications" and "protocols", each a list of category groups @@ -322,10 +206,52 @@ def list_appgroup_catalog() -> dict[str, list[dict]]: - Exception: if the engine cannot be queried or a catalog is missing """ loaded = {'applications': load_applications(), 'protocols': load_protocols()} - return { - kind: __catalog_groups(__catalog_entries(__load_catalog(filename), loaded[kind], kind)) - for kind, filename in APPGROUP_CATALOG_FILES.items() - } + result = {} + for kind, filename in APPGROUP_CATALOG_FILES.items(): + catalog = __load_catalog(filename) + # (category tag, category label, item), in catalog order + items = [] + for entry in catalog: + # an application marked inactive, or a protocol marked deprecated, is gone from the product + if kind == 'applications' and entry.get('active') is False: + continue + if kind == 'protocols' and entry.get('deprecated'): + continue + label = entry.get('label', '') + # "unknown" is the engine's catch-all for what it could not classify, not a group member + if kind == 'protocols' and label.lower() == 'unknown': + continue + engine_name = loaded[kind].get(entry.get('id')) + category = entry.get('category') or {} + item = {'id': engine_name or entry.get('tag', ''), 'label': label, 'selectable': engine_name is not None} + if entry.get('icon'): + item['logo'] = entry.get('icon') + items.append((category.get('tag', UNCATEGORIZED_TAG), category.get('label', ''), item)) + + # what the engine loaded but the catalog does not list is still matchable: kept, named after the + # engine and left uncategorised + known = {entry.get('id') for entry in catalog} + for entry_id, name in loaded[kind].items(): + if entry_id in known: + continue + if kind == 'protocols' and name.lower() == 'unknown': + continue + items.append((UNCATEGORIZED_TAG, '', {'id': name, 'label': name, 'selectable': True})) + + groups = dict[str, dict]() + for tag, label, item in items: + group = groups.get(tag) + if group is None: + group = groups[tag] = {'tag': tag, 'label': label, 'selectable': tag != UNCATEGORIZED_TAG, 'items': []} + group['items'].append(item) + + for group in groups.values(): + group['items'].sort(key=lambda item: item['label'].lower()) + + result[kind] = sorted(groups.values(), key=lambda group: (group['tag'] == UNCATEGORIZED_TAG, + group['label'].lower())) + + return result def __validate_appgroup_members(kind: str, values: list[str]) -> list[str]: @@ -855,7 +781,6 @@ def add_rule(e_uci: EUci, name: str, enabled: bool, action: str, source: list[st order.insert(0, config_name) if position == 'top' else order.append(config_name) __renumber_rules(e_uci, order) - __toggle_engine(e_uci) e_uci.save('dpi') return config_name @@ -897,7 +822,6 @@ def edit_rule(e_uci: EUci, config_name: str, name: str, enabled: bool, action: s appgroups = __validate_appgroups(e_uci, appgroups, match_all) __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups, match_all) - __toggle_engine(e_uci) e_uci.save('dpi') return config_name @@ -921,7 +845,6 @@ def delete_rule(e_uci: EUci, config_name: str) -> str: e_uci.delete('dpi', config_name) __renumber_rules(e_uci) - __toggle_engine(e_uci) e_uci.save('dpi') return config_name @@ -967,7 +890,6 @@ def enable_rule(e_uci: EUci, config_name: str) -> str: raise ValidationError('id', 'rule_not_found', config_name) e_uci.set('dpi', config_name, 'enabled', '1') - __toggle_engine(e_uci) e_uci.save('dpi') return config_name @@ -990,7 +912,6 @@ def disable_rule(e_uci: EUci, config_name: str) -> str: raise ValidationError('id', 'rule_not_found', config_name) e_uci.set('dpi', config_name, 'enabled', '0') - __toggle_engine(e_uci) e_uci.save('dpi') return config_name @@ -1110,7 +1031,7 @@ def __exemption_source(e_uci: EUci, criteria: str) -> list[str] | None: return None -def migrate_schema(e_uci: EUci) -> bool: +def migrate_schema(e_uci: EUci): """ Migrate every rule and exemption from the schema used before application groups existed. @@ -1122,16 +1043,15 @@ def migrate_schema(e_uci: EUci) -> bool: Each global exemption becomes an Allow rule at the top of the list: a managed match-all rule narrowed to a source when its criteria is a plain address, CIDR or firewall object; an unmanaged one carrying the criteria verbatim otherwise. Disabled exemptions become disabled rules. - The `exemption` section type, `firewall_exemption` and `popular_filters` are then removed. + The `exemption` section type and `popular_filters` are then removed. - Safe to call unconditionally on every boot: a box with nothing left in the old schema returns False + The changes are committed and the dpi service reloaded right away. + + Safe to call unconditionally on every boot: a box with nothing left in the old schema returns without touching UCI. Args: - e_uci: euci instance - - Returns: - True if the config was changed (and therefore committed and reloaded), False otherwise """ exemptions = utils.get_all_by_type(e_uci, 'dpi', 'exemption') or {} rules = utils.get_all_by_type(e_uci, 'dpi', 'rule') or {} @@ -1140,7 +1060,7 @@ def migrate_schema(e_uci: EUci) -> bool: if any(field in rule for field in legacy_fields)} if not exemptions and not legacy_rules: - return False + return order = [] @@ -1192,9 +1112,7 @@ def migrate_schema(e_uci: EUci) -> bool: order.append(section) __renumber_rules(e_uci, order) - e_uci.delete('dpi', 'config', 'firewall_exemption') e_uci.delete('dpi', 'config', 'popular_filters') - __toggle_engine(e_uci) e_uci.save('dpi') - __apply(e_uci) - return True + e_uci.commit('dpi') + subprocess.run(["/etc/init.d/dpi", "reload"], check=True) diff --git a/packages/python3-nethsec/src/nethsec/inventory/__init__.py b/packages/python3-nethsec/src/nethsec/inventory/__init__.py index 413c2d6c9..ae370128f 100644 --- a/packages/python3-nethsec/src/nethsec/inventory/__init__.py +++ b/packages/python3-nethsec/src/nethsec/inventory/__init__.py @@ -642,10 +642,10 @@ def fact_proxy_pass(uci: EUci): def fact_dpi(uci: EUci): ret = {"enabled": False, "rules": 0} - ret["enabled"] = uci.get('dpi', 'config', 'enabled', default='0') == '1' for rule in utils.get_all_by_type(uci, 'dpi', 'rule'): if uci.get('dpi', rule, 'enabled', default='0') == '1': ret["rules"] += 1 + ret["enabled"] = ret["rules"] > 0 return ret def fact_extra_packages(uci: EUci): diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py index e71a99f49..ff5d5c72c 100644 --- a/packages/python3-nethsec/tests/test_dpi.py +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -1,4 +1,5 @@ import pathlib +import subprocess import pytest from euci import EUci @@ -55,14 +56,12 @@ config main 'config' option log_blocked '0' option firewall_exemption '0' - option enabled '0' """ dpi_db = """ config main 'config' option log_blocked '0' option firewall_exemption '0' - option enabled '0' list popular_filters 'netify.netflix' list popular_filters 'netify.hulu' list popular_filters 'netify.whatsapp' @@ -188,13 +187,31 @@ """ +DPI_RELOAD = ["/etc/init.d/dpi", "reload"] + + @pytest.fixture(autouse=True) -def mock_apply(mocker: MockFixture): +def mock_reload(mocker: MockFixture): """ - add/delete of rules and appgroups commit the dpi config and reload the dpi service right away; - tests only care about the resulting uci state, not about actually reloading a system service. + The schema migration reloads the dpi service right away; tests only care about the resulting uci + state, not about actually reloading a system service. Every other command runs as usual. """ - return mocker.patch('nethsec.dpi.__apply') + run = subprocess.run + return mocker.patch('nethsec.dpi.subprocess.run', + side_effect=lambda args, *a, **kw: None if args == DPI_RELOAD else run(args, *a, **kw)) + + +def reloaded(mock_reload) -> bool: + return any(call.args and call.args[0] == DPI_RELOAD for call in mock_reload.call_args_list) + + +def has_pending_changes(e_uci: EUci) -> bool: + delta = pathlib.Path(e_uci.savedir()).joinpath('dpi') + return delta.exists() and delta.stat().st_size > 0 + + +def dpi_config_file(e_uci: EUci) -> str: + return pathlib.Path(e_uci.confdir()).joinpath('dpi').read_text() @pytest.fixture @@ -360,7 +377,7 @@ def test_add_appgroup_matches_members_case_insensitively(e_uci_appgroups): def test_add_appgroup_skips_validation_without_a_vocabulary(e_uci_with_data, mocker): mocker.patch('nethsec.dpi.load_applications', side_effect=FileNotFoundError) - mocker.patch('nethsec.dpi.__load_catalog_tags', side_effect=FileNotFoundError) + mocker.patch('nethsec.dpi.__load_catalog', side_effect=FileNotFoundError) group_id = dpi.add_appgroup(e_uci_with_data, 'Group one', ['netify.unknown-to-everything']) assert e_uci_with_data.get('dpi', group_id, 'app', list=True) == ('netify.unknown-to-everything',) @@ -398,16 +415,17 @@ def test_delete_appgroup(e_uci_appgroups): assert e_uci_appgroups.get('dpi', group_id, default=None) is None -def test_add_appgroup_leaves_the_change_pending(e_uci_appgroups, mock_apply): +def test_add_appgroup_leaves_the_change_pending(e_uci_appgroups, mock_reload): dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) - mock_apply.assert_not_called() + assert has_pending_changes(e_uci_appgroups) + assert not reloaded(mock_reload) -def test_delete_appgroup_leaves_the_change_pending(e_uci_appgroups, mock_apply): +def test_delete_appgroup_leaves_the_change_pending(e_uci_appgroups, mock_reload): group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) - mock_apply.reset_mock() dpi.delete_appgroup(e_uci_appgroups, group_id) - mock_apply.assert_not_called() + assert has_pending_changes(e_uci_appgroups) + assert not reloaded(mock_reload) def test_delete_appgroup_of_unknown_id(e_uci_appgroups): @@ -683,12 +701,6 @@ def test_add_rule_without_source(e_uci_rules): assert e_uci_rules.get('dpi', rule_id, 'source', default=None) is None -def test_add_rule_enables_the_engine(e_uci_rules): - group = group_of(e_uci_rules) - dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) - assert e_uci_rules.get('dpi', 'config', 'enabled') == '1' - - def test_add_rule_at_the_top(e_uci_rules): group = group_of(e_uci_rules) first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) @@ -789,19 +801,19 @@ def test_delete_rule_closes_the_gap(e_uci_rules): assert e_uci_rules.get('dpi', third, 'priority') == '2' -def test_add_rule_leaves_the_change_pending(e_uci_rules, mock_apply): +def test_add_rule_leaves_the_change_pending(e_uci_rules, mock_reload): group = group_of(e_uci_rules) - mock_apply.reset_mock() dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) - mock_apply.assert_not_called() + assert has_pending_changes(e_uci_rules) + assert not reloaded(mock_reload) -def test_delete_rule_leaves_the_change_pending(e_uci_rules, mock_apply): +def test_delete_rule_leaves_the_change_pending(e_uci_rules, mock_reload): group = group_of(e_uci_rules) rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) - mock_apply.reset_mock() dpi.delete_rule(e_uci_rules, rule_id) - mock_apply.assert_not_called() + assert has_pending_changes(e_uci_rules) + assert not reloaded(mock_reload) def test_delete_rule_of_unknown_id(e_uci_rules): @@ -810,13 +822,6 @@ def test_delete_rule_of_unknown_id(e_uci_rules): assert err.value.args[1] == 'rule_not_found' -def test_delete_last_enabled_rule_disables_the_engine(e_uci_rules): - group = group_of(e_uci_rules) - rule_id = dpi.add_rule(e_uci_rules, 'Only one', True, 'block', [], [group]) - dpi.delete_rule(e_uci_rules, rule_id) - assert e_uci_rules.get('dpi', 'config', 'enabled') == '0' - - def test_rename_and_toggle_rule(e_uci_rules): group = group_of(e_uci_rules) rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) @@ -824,10 +829,8 @@ def test_rename_and_toggle_rule(e_uci_rules): assert e_uci_rules.get('dpi', rule_id, 'ns_name') == 'Renamed' dpi.disable_rule(e_uci_rules, rule_id) assert e_uci_rules.get('dpi', rule_id, 'enabled') == '0' - assert e_uci_rules.get('dpi', 'config', 'enabled') == '0' dpi.enable_rule(e_uci_rules, rule_id) assert e_uci_rules.get('dpi', rule_id, 'enabled') == '1' - assert e_uci_rules.get('dpi', 'config', 'enabled') == '1' def test_rename_and_toggle_work_on_unmanaged_rules(e_uci_with_data): @@ -1043,7 +1046,6 @@ def test_freeze_legacy_criteria_applies_vlan_rewrite(e_uci: EUci): config main 'config' option log_blocked '0' option firewall_exemption '1' - option enabled '0' list popular_filters 'netify.facebook' config rule 'ns_b01a0e73' @@ -1095,13 +1097,16 @@ def e_uci_legacy(e_uci: EUci): return e_uci -def test_migrate_schema_noop_when_nothing_to_migrate(e_uci, mock_apply): - assert dpi.migrate_schema(e_uci) is False - mock_apply.assert_not_called() +def test_migrate_schema_noop_when_nothing_to_migrate(e_uci, mock_reload): + config_before = dpi_config_file(e_uci) + dpi.migrate_schema(e_uci) + assert dpi_config_file(e_uci) == config_before + assert not has_pending_changes(e_uci) + assert not reloaded(mock_reload) def test_migrate_schema_freezes_legacy_rule(e_uci_legacy): - assert dpi.migrate_schema(e_uci_legacy) is True + dpi.migrate_schema(e_uci_legacy) rule = e_uci_legacy.get_all('dpi', 'ns_b01a0e73') assert rule['criteria'] == ( "(iface_nfq_src == 'eth0' or iface_nfq_dst == 'eth0') && " @@ -1173,7 +1178,6 @@ def test_migrate_schema_converts_object_exemption_via_expanded_addresses(e_uci, with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: fp.write(""" config main 'config' - option enabled '0' config exemption 'ns_objex' option enabled '1' @@ -1182,7 +1186,7 @@ def test_migrate_schema_converts_object_exemption_via_expanded_addresses(e_uci, """) mocker.patch('nethsec.dpi.objects.is_object_id', return_value=True) mocker.patch('nethsec.dpi.objects.get_object_ips', return_value=['192.168.50.0/24']) - assert dpi.migrate_schema(e_uci) is True + dpi.migrate_schema(e_uci) rules = dpi.list_rules(e_uci) assert len(rules) == 1 assert rules[0]['managed'] is True @@ -1192,7 +1196,8 @@ def test_migrate_schema_converts_object_exemption_via_expanded_addresses(e_uci, def test_migrate_schema_removes_exemption_sections_and_legacy_globals(e_uci_legacy): dpi.migrate_schema(e_uci_legacy) assert utils.get_all_by_type(e_uci_legacy, 'dpi', 'exemption') == {} - assert e_uci_legacy.get('dpi', 'config', 'firewall_exemption', default=None) is None + # still read by dpi-config and the hotplug script, so the migration must leave it alone + assert e_uci_legacy.get('dpi', 'config', 'firewall_exemption') == '1' assert e_uci_legacy.get('dpi', 'config', 'popular_filters', list=True, default=None) is None @@ -1203,13 +1208,17 @@ def test_migrate_schema_puts_exemptions_before_migrated_rules(e_uci_legacy): assert names_in_order.index('Migrated exception 1') < names_in_order.index('Migrated rule 1') -def test_migrate_schema_applies_immediately(e_uci_legacy, mock_apply): - assert dpi.migrate_schema(e_uci_legacy) is True - mock_apply.assert_called_with(e_uci_legacy) +def test_migrate_schema_applies_immediately(e_uci_legacy, mock_reload): + dpi.migrate_schema(e_uci_legacy) + assert not has_pending_changes(e_uci_legacy) + assert reloaded(mock_reload) -def test_migrate_schema_is_idempotent(e_uci_legacy): +def test_migrate_schema_is_idempotent(e_uci_legacy, mock_reload): + dpi.migrate_schema(e_uci_legacy) + config_after_first_run = dpi_config_file(e_uci_legacy) + mock_reload.reset_mock() dpi.migrate_schema(e_uci_legacy) - rules_after_first_run = dpi.list_rules(e_uci_legacy) - assert dpi.migrate_schema(e_uci_legacy) is False - assert dpi.list_rules(e_uci_legacy) == rules_after_first_run + assert dpi_config_file(e_uci_legacy) == config_after_first_run + assert not has_pending_changes(e_uci_legacy) + assert not reloaded(mock_reload) diff --git a/packages/python3-nethsec/tests/test_inventory.py b/packages/python3-nethsec/tests/test_inventory.py index beca36850..ca151c811 100644 --- a/packages/python3-nethsec/tests/test_inventory.py +++ b/packages/python3-nethsec/tests/test_inventory.py @@ -561,7 +561,6 @@ dpi_db = """ config main 'config' option log_blocked '0' - option enabled '1' option firewall_exemption '0' config rule 'ns_2b170d05' @@ -928,6 +927,11 @@ def test_fact_ipsec(tmp_path): def test_fact_dpi(tmp_path): u = _setup_db(tmp_path) assert inventory.fact_dpi(u) == {"enabled": True, "rules": 1} + +def test_fact_dpi_without_enabled_rules(tmp_path): + u = _setup_db(tmp_path) + u.set('dpi', 'ns_2b170d05', 'enabled', '0') + assert inventory.fact_dpi(u) == {"enabled": False, "rules": 0} def test_fact_dhcp_server(tmp_path): u = _setup_db(tmp_path) From d57ef5932ab677f9501c6381d537ad069923e957 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Fri, 2 Oct 2026 18:01:58 +0200 Subject: [PATCH 13/17] refactor(dpi): drop legacy config paths and trim docs --- .../netifyd/references/nethsecurity-wiring.md | 51 ++-- packages/ns-api/README.md | 186 ++++++-------- packages/ns-dpi/Makefile | 3 +- packages/ns-dpi/README.md | 232 ++++++------------ packages/ns-dpi/files/20_dpi | 6 +- .../ns-dpi/files/22_dpi_firewall_exemption | 14 -- packages/ns-dpi/files/23_dpi_drop_enabled | 14 -- packages/ns-dpi/files/23_dpi_drop_options | 21 ++ packages/ns-dpi/files/dpi | 6 + packages/ns-dpi/files/dpi-config | 131 +++------- packages/ns-dpi/files/dpi-exemption.hotplug | 10 +- .../src/nethsec/dpi/__init__.py | 24 +- packages/python3-nethsec/tests/test_dpi.py | 50 +++- .../python3-nethsec/tests/test_inventory.py | 1 - 14 files changed, 320 insertions(+), 429 deletions(-) delete mode 100755 packages/ns-dpi/files/22_dpi_firewall_exemption delete mode 100755 packages/ns-dpi/files/23_dpi_drop_enabled create mode 100755 packages/ns-dpi/files/23_dpi_drop_options diff --git a/.agents/skills/netifyd/references/nethsecurity-wiring.md b/.agents/skills/netifyd/references/nethsecurity-wiring.md index 31b9b8c1c..9c0870344 100644 --- a/.agents/skills/netifyd/references/nethsecurity-wiring.md +++ b/.agents/skills/netifyd/references/nethsecurity-wiring.md @@ -83,32 +83,44 @@ lists. `procd_set_param file /etc/netifyd.conf` means procd restarts the service `/usr/sbin/dpi` is the apply path, in order: -1. `/usr/sbin/dpi-config` — reads `/etc/config/dpi`, writes `/etc/netifyd/netify-proc-flow-actions.json` -2. `/usr/sbin/dpi-nft` — writes the `dpi_actions` / `dpi_dummy` chains into +1. `/usr/libexec/ns-dpi/dpi-migrate` — converts a `/etc/config/dpi` still on the pre-appgroup schema + (`dpi.migrate_schema`, a no-op when there is nothing to migrate). It also runs from the + `21_dpi_migrate` uci-default; running it here covers restored backups and configs synced by an HA + peer, where uci-defaults don't run +2. `/usr/sbin/dpi-config` — reads `/etc/config/dpi`, writes `/etc/netifyd/netify-proc-flow-actions.json` +3. `/usr/sbin/dpi-nft` — writes the `dpi_actions` / `dpi_dummy` chains into `/usr/share/nftables.d/table-pre/` -3. `/etc/init.d/netifyd reload` +4. `/etc/init.d/netifyd reload` + +Steps 1 and 2 run under `/var/lock/dpi.lock`, shared with the iface hotplug, so the migration and the +generation never run twice at once. Current generator behaviour worth knowing before touching it: -- `valid_actions` is `block`, `bulk`, `best_effort`, `video`, `voice`; anything else, or a disabled rule, - is skipped. -- A rule's raw `criteria` takes precedence over generated source/application/protocol/category matching. - Double quotes in it are rewritten to single quotes. **No `;` is appended to raw criteria** — generated - criteria get one, raw ones don't. Same for entries in the global `exemptions` array. -- A `device` naming a VLAN device causes `vlan_id == N && ` to be prepended — including to raw criteria, - so a rule carrying both its own `criteria` and a `device` gets the term whether or not it already has one. -- A hidden `analyzed` action (`detection_guessed || detection_complete;`) is appended last, labelling every - analysed flow. -- `dpi.config.firewall_exemption` (default `1`, forced to `1` on every upgrade by `22_dpi_firewall_exemption`) - pre-fills `exemptions` with every firewall interface IP; `/etc/hotplug.d/iface/90-dpi-exemption` re-runs - `dpi-config` on `ifup` / address `ifupdate` and does `netifyd reload` only if the JSON changed and the - agent is running (so HA backup nodes are left alone); - `exemption` sections add more, and an exemption whose criteria is an object ID is expanded to its IPs. +- The first action is a hidden `analyzed` one, generated by `dpi-config` and never stored in UCI: + priority `0`, `halt_on_match: false`, criteria `detection_guessed || detection_complete;`. It labels every + analysed flow and lets the evaluation continue to the user rules. +- User rules are emitted with their UCI `priority` (dense, from `1`) and `halt_on_match: true`: the first + matching rule wins. Actions are `block` and `allow`; `allow` sets a label no nft rule acts on, so it only + shields its flows from the block rules below it. A rule with no `enabled` option is skipped. +- Rules without a raw `criteria` (the ones created by the API) get it from `dpi.build_rule_criteria`: + `source` ORed, ANDed with the expanded `appgroup` members; `ns_match_all` with no source becomes `*`. +- Rules with a raw `criteria` (hand-written, or frozen by the migration) have double quotes rewritten to + single quotes and `;` appended when missing: an unterminated expression is a syntax error. +- A rule that would emit an empty criteria is skipped, because an empty criteria takes the agent down. +- A rule still on the pre-appgroup schema (`device`, `application`, `protocol` or `category` set) is + skipped with a warning, never emitted: read as a current rule, its `source` alone would block every flow + of those hosts. The migration in step 1 converts it first. +- The global `exemptions` are always pre-filled with every firewall interface IP, so DPI rules can not + lock the administrator out; `/etc/hotplug.d/iface/90-dpi-exemption` re-runs `dpi-config` on `ifup` / + address `ifupdate` and does `netifyd reload` only if the JSON changed and the agent is running (so HA + backup nodes are left alone). They are the only entries in `exemptions`. - `dpi.config.log_blocked` drives the nft log rule in `dpi-nft`, not any netify logging target. Enforcement is nft reading conntrack labels, not netifyd acting directly: netifyd classifies and sets a -label via its `ctlabel` target, and `dpi_actions` (hook prerouting, `filter + 10`) rejects labelled -traffic (`netify-blocked`) or DSCP-marks it (the QoS labels — the DPI→qosify bridge). +label via its `ctlabel` target, and `dpi_actions` (hook prerouting, `filter + 10`) rejects the flows +labelled `netify-blocked`. The chain still DSCP-marks the QoS labels, which no DPI action sets any more: +they are the DPI→qosify bridge, left in place. ## Conntrack labels @@ -124,6 +136,7 @@ across an upgrade keep their labels, so **never reassign an existing bit** — t | 4 | `best_effort` | | 5 | `video` | | 6 | `voice` | +| 7 | `netify-allowed` | `netify-analyzed` is consumed by no nft rule and read only by `ns.flows` for display — it is a diagnostic that tells support whether netifyd actually analysed a flow. "Nothing reads it in code" is not a reason to diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index c2673e8d3..0c5ea2b03 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -3548,12 +3548,31 @@ Response example: ## ns.dpi -Manage netifyd DPI engine. +Manage DPI rules and application groups. Rules are evaluated in priority order and the first rule +matching a flow wins. + +### list-application-catalog + +Return the raw application catalog, `/etc/netifyd/netify-application-catalog.json`: + +```bash +api-cli ns.dpi list-application-catalog +``` + +The response is `{"values": [...]}`, with the catalog entries as downloaded; `{}` if the catalog is +missing. + +### list-protocol-catalog + +Same as `list-application-catalog`, for `/etc/netifyd/netify-protocol-catalog.json`: + +```bash +api-cli ns.dpi list-protocol-catalog +``` ### list-appgroup-catalog -List everything an application group can be built with: the catalogs crossed with what the netifyd -engine has loaded, grouped by category, done once on the firewall directly. +List the applications and protocols an application group can be built with, grouped by category: ```bash api-cli ns.dpi list-appgroup-catalog @@ -3614,46 +3633,18 @@ Example response: } ``` -**`selectable` on an item** means the engine has the signature loaded, so the item can be matched on -this machine. Unregistering a subscription strips the premium signatures and drops the matchable set -from ~2600 applications to ~200, while the catalog still lists them all: catalog presence is not -matchability. +- item `selectable`: the engine has the signature loaded, so the item can be used in a group. Without a + subscription most applications are not selectable +- item `id`: the value to pass to `add-appgroup`; only valid when `selectable` is `true` +- group `selectable`: the category can be used as a group member (`application_categories` or + `protocol_categories`). The group with an empty `tag` collects uncategorised items and is never + selectable as a whole +- `logo` is omitted when the catalog has no icon +- groups and items are sorted by label, the uncategorised group last -**`id` is only submittable when `selectable` is `true`.** It carries the engine name whenever there is -one, which is what `add-appgroup` stores and what a criteria matches on. For an item the engine has not -loaded there is no engine name, so the catalog tag stands in — and the two differ, `ftp-control` in the -catalog against `FTP` in the engine. Sending the id of a non-selectable item is refused with -`invalid_application` or `invalid_protocol`. +Inactive applications, deprecated protocols and the `unknown` protocol are left out. -**`selectable` on a group** means the category itself can be a group member, passed as -`application_categories` or `protocol_categories` to `add-appgroup`. Every real category can; the group -with an empty `tag` is the bucket holding whatever the catalog does not categorise, it is not a category -and no criteria can match on it, so it is never selectable as a whole. Its items are selectable -individually like any other. - -**`label` is English**, the only language the catalog carries, and `logo` is absent when the catalog has -no icon for the entry — always for protocols. A client showing translated category names should -translate by `tag` and keep `label` as the fallback, then sort the groups again: they arrive sorted by -their English label, with the uncategorised group last. - -What is left out, and why: - -- applications the catalog marks `active: false` and protocols it marks deprecated: they are gone from - the product -- the `unknown` protocol: it is the engine's catch-all for what it could not classify -- nothing else — what the engine loaded but the catalog does not list is kept, named after the engine - and left uncategorised, because it is matchable all the same - -Error response: - -```json -{"error": "catalog_not_available"} -``` - -Returned whenever any source is missing: the engine is not running, or a catalog has not been downloaded -yet. The answer is all-or-nothing on purpose — a partial one would quietly offer a smaller product than -the machine actually has. The catalogs are refreshed nightly by `dpi-data-update` into `/etc/netifyd`, so -a machine that has never reached `distfeed.nethesis.it` has none. +The call fails when the engine is not running or a catalog has not been downloaded yet. ### list-appgroups @@ -3663,16 +3654,15 @@ List the application groups, ordered by name: api-cli ns.dpi list-appgroups ``` -Data can be limited and paginated with the `limit` and `page` parameters, and filtered by name with -`search`: +Optional parameters: `search` (matched against the name), `limit` and `page`. Without `limit` every group +is returned: ```bash api-cli ns.dpi list-appgroups --data '{"search": "business", "limit": 10, "page": 2}' ``` -Without `limit` every group is returned, which is what the rule drawer needs to fill its group selector. - Example response: + ```json { "values": { @@ -3696,7 +3686,7 @@ Example response: } ``` -`used` and `matches` report the rules referencing the group: a group in use can't be deleted. +`used` and `matches` list the rules referencing the group. ### add-appgroup @@ -3713,21 +3703,16 @@ api-cli ns.dpi add-appgroup --data '{ ``` - `name`: mandatory, up to 64 characters, unique among the groups regardless of case -- the four member lists are all optional, but the group must hold at least one member overall -- applications and protocols are named as the **DPI engine** reports them, which is the `id` of a - selectable item of `list-appgroup-catalog`, **not** the `tag` of the raw catalogs. For applications - the two coincide (`netify.amazon`); for protocols they don't (`HTTP/Connect` in the engine, - `http-connect` in the catalog) and only the engine name can be matched -- categories are named by tag (`cybersecurity`, `games`), which is the same in the engine and in the - catalogs. Application and protocol categories are distinct vocabularies - -Members are validated against what the engine has loaded, with the catalog accepted as well for -applications, so that a group stays editable on a machine whose premium signatures were removed. When -neither source can be read the values are stored unchecked: an unknown value never matches, so refusing -the write would be worse than accepting it. Values are always rejected when empty or when they hold a -character that could break the generated expression. +- `applications`, `protocols`: names as reported by the engine, i.e. the `id` of a selectable item of + `list-appgroup-catalog` (`HTTP/Connect`, not the catalog tag `http-connect`) +- `application_categories`, `protocol_categories`: category tags +- the four lists are optional, but the group must have at least one member + +Members are checked against the engine and the downloaded catalogs; the check is skipped when neither is +available. Example response: + ```json { "id": "ns_1a2b3c4d5" @@ -3747,6 +3732,7 @@ api-cli ns.dpi edit-appgroup --data '{ ``` Example response: + ```json { "id": "ns_1a2b3c4d5" @@ -3761,41 +3747,31 @@ Delete an application group: api-cli ns.dpi delete-appgroup --data '{"id": "ns_1a2b3c4d5"}' ``` -Deleting a group referenced by a rule is refused with the list of the referencing rules: -```json -{ - "validation": { - "errors": [ - { - "parameter": "id", - "message": "appgroup_is_used", - "value": ["dpi/ns_9f8e7d6c5"] - } - ] - } -} -``` - Example response: + ```json { "message": "success" } ``` +A group referenced by a rule can't be deleted. + ### list-rules -List every rule in priority order, which is the order they are evaluated in: the first rule matching a -flow wins and stops the evaluation. +List the rules in priority order: ```bash api-cli ns.dpi list-rules ``` -The list is never paginated nor filtered: it is the whole set, which is also what `order-rules` needs. -Rules hidden with `ns_visible '0'` are the only ones left out. `managed` tells whether the rule was -created through the API: an unmanaged rule can be renamed, enabled, reordered and deleted, but not -edited, and it carries the raw `criteria` it matches on. `match_all` tells whether the rule matches every flow instead of naming application groups. +Rules with `ns_visible '0'` are left out. + +- `managed`: the rule was created through the API. Unmanaged rules, e.g. the ones migrated from the + previous schema, carry their raw `criteria` and can be renamed, enabled, disabled, reordered and + deleted, but not edited +- `match_all`: the rule matches every flow instead of naming application groups +- `index`: position of the rule in the list Example response: @@ -3831,7 +3807,7 @@ Example response: }, { "id": "ns_f1c6e9e0", - "name": "Legacy rule", + "name": "Migrated rule 1", "enabled": true, "action": "block", "source": [], @@ -3847,7 +3823,7 @@ Example response: ### add-rule -Add a DPI rule: +Add a rule: ```bash api-cli ns.dpi add-rule --data '{ @@ -3861,28 +3837,17 @@ api-cli ns.dpi add-rule --data '{ }' ``` -Parameters: - - `name`: mandatory, up to 64 characters -- `enabled`: `true` or `false` -- `action`: `block` or `allow`. `allow` blocks nothing: it labels the flow and, since every rule halts - on match, stops the evaluation before any block rule below it can fire -- `source`: list of addresses, networks or ranges, IPv4 and IPv6 alike, e.g. - `["192.168.1.1", "192.168.1.0/24", "10.0.0.10-10.0.0.20"]`. An empty list matches every host. Ranges - are expanded to CIDR blocks when the rule is generated -- `appgroups`: config names of the application groups the rule matches, refer to `list-appgroups`. **At - least one is required unless `match_all` is set**: a rule with a source and no group would be an - IP-level rule, which the firewall does better, and a rule with neither would match nothing -- `match_all`: `true` makes the rule match every flow, whatever the application. `appgroups` must then - be empty, and passing both is refused with `appgroups_not_allowed_with_match_all`. Combined with a - `source`, it means every flow of those hosts. Defaults to `false` -- `position`: `top` to evaluate the rule before every other one, `bottom` (the default) after them - -There is no device or interface parameter: per-interface rules are not reachable from the API. - -A `match_all` rule placed at the top of the list shadows every rule below it, because every rule halts -on match. That is the point of an `allow` one — it is how a bypass is expressed — but a client offering -it should make the consequence visible. +- `enabled`: mandatory, `true` or `false` +- `action`: mandatory, `block` or `allow`. `allow` stops the evaluation, so the rules below it don't apply + to the matching flows +- `source`: addresses, networks or ranges, IPv4 or IPv6, e.g. + `["192.168.1.1", "192.168.1.0/24", "10.0.0.10-10.0.0.20"]`. Empty or omitted matches every host +- `appgroups`: ids of the application groups to match, see `list-appgroups`. Mandatory unless + `match_all` is `true` +- `match_all`: match every flow, optionally narrowed by `source`. `appgroups` must be empty. Defaults to + `false` +- `position`: `top` or `bottom` of the list. Defaults to `bottom` Example response: @@ -3894,8 +3859,8 @@ Example response: ### edit-rule -Same payload as `add-rule` minus `position`, plus the `id` of the rule. The priority is left untouched, -use `order-rules` to move the rule. +Same payload as `add-rule` without `position`, plus the `id` of the rule. The priority is not changed, +use `order-rules` to move the rule. Only managed rules can be edited: ```bash api-cli ns.dpi edit-rule --data '{ @@ -3909,9 +3874,6 @@ api-cli ns.dpi edit-rule --data '{ }' ``` -Only managed rules can be edited: editing a rule carrying a hand-written criteria is refused with -`rule_not_managed`, because it has no source and no group to rebuild it from. - Example response: ```json @@ -3922,7 +3884,7 @@ Example response: ### delete-rule -Delete a DPI rule. The priorities of the remaining rules are renumbered, so no gap is left behind: +Delete a rule. The remaining rules are renumbered: ```bash api-cli ns.dpi delete-rule --data '{"id": "ns_f1c6e9e0"}' @@ -3986,15 +3948,13 @@ Example response: ### order-rules -Reorder the rules, renumbering their priorities from the given order: +Set the priority of the rules from the given order: ```bash api-cli ns.dpi order-rules --data '{"order": ["ns_f1c6e9e0", "ns_3869dc35"]}' ``` -The order must name **every** rule exactly once, hidden ones included, as `list-rules` returns them. A -partial order is refused with `invalid_order`: silently moving the rules left out is never what the -caller meant. +`order` must list every rule exactly once, including the ones with `ns_visible '0'`. Example response: diff --git a/packages/ns-dpi/Makefile b/packages/ns-dpi/Makefile index 06e8e5f6e..59ccb8567 100644 --- a/packages/ns-dpi/Makefile +++ b/packages/ns-dpi/Makefile @@ -83,8 +83,7 @@ define Package/ns-dpi/install $(INSTALL_BIN) ./files/dpi-migrate.py $(1)/usr/libexec/ns-dpi/dpi-migrate $(INSTALL_CONF) ./files/20_dpi $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/21_dpi_migrate $(1)/etc/uci-defaults/21_dpi_migrate - $(INSTALL_BIN) ./files/22_dpi_firewall_exemption $(1)/etc/uci-defaults/22_dpi_firewall_exemption - $(INSTALL_BIN) ./files/23_dpi_drop_enabled $(1)/etc/uci-defaults/23_dpi_drop_enabled + $(INSTALL_BIN) ./files/23_dpi_drop_options $(1)/etc/uci-defaults/23_dpi_drop_options $(INSTALL_BIN) ./files/99-dpi-license-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-license-update-cron $(INSTALL_BIN) ./files/99-dpi-data-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-data-update-cron $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/register diff --git a/packages/ns-dpi/README.md b/packages/ns-dpi/README.md index c4027a014..fc308a832 100644 --- a/packages/ns-dpi/README.md +++ b/packages/ns-dpi/README.md @@ -1,183 +1,113 @@ # ns-dpi -Manage network traffic using DPI on network flows. +Block traffic by application and protocol using the netifyd DPI engine. How it works: -- Netify flow actions plugin adds a label to matching connections -- nft rules can block or change priority (`dscp`) to connections with labels - -netifyd runs all the time: traffic is filtered as soon as at least one rule is enabled. -To apply a change made by hand to the `dpi` UCI database (see below for an example): +- `dpi-config` turns `/etc/config/dpi` into the netifyd flow actions config, + `/etc/netifyd/netify-proc-flow-actions.json` +- netifyd sets a conntrack label on the flows matching a rule +- the `dpi_actions` nft chain, written by `dpi-nft` in `/usr/share/nftables.d/table-pre/`, rejects the + flows labelled `netify-blocked` + +netifyd runs all the time: traffic is filtered as soon as one rule is enabled. +The firewall IP addresses are always excluded from the DPI rules, so a rule can not block access to the +firewall itself; addresses changing at runtime (DHCP, PPPoE) are tracked by +`/etc/hotplug.d/iface/90-dpi-exemption`. +Rules and application groups are managed by the `ns.dpi` API, see `packages/ns-api/README.md`. +After editing `/etc/config/dpi` by hand, apply the changes with: ``` uci commit dpi /etc/init.d/dpi reload ``` -Global options: - -- `log_blocked`: can be `0` or `1`, if set to `1` blocked connections will be logged inside `/var/log/messages` -- `firewall_exemption`: can be `0` or `1`, if set to `1` all firewall IP addresses will be - added to global exemption list and will not match DPI rules; default is `1`, so DPI rules can not block - access to the firewall itself (traffic forwarded between hosts is still filtered). - The option is forced to `1` on every image upgrade, so a `0` set by hand does not survive an upgrade. - Addresses changing at runtime (DHCP, PPPoE) are tracked by `/etc/hotplug.d/iface/90-dpi-exemption`: on - `ifup` or an address `ifupdate` it regenerates the flow actions config and, only if it changed and netifyd - is running, reloads netifyd -- `popular_filters`: list of filters that will be returned to from `api-cli ns.dpi list-popular` call. -- `ns_exclude`: list of network interface exclusions in Netifyd that will be returned by `uci show netifyd.@netifyd[0].ns_exclude` - -Rule options: - -- `criteria`: DPI expression to match the traffic - - the criteria must terminate with `;` when using complex expressions - - use the `"` symbol to enclose strings, double-qoutes will be then translated to `'` inside the plugin configuration file (`/etc/netifyd/netify-flow-actions.json`) -- `source`: match all traffic from the given address, it accepts also an object like `/`; when using an object make sure to not use complex criteria -- `action`: valid actions are: - - `block`: matching traffic will be blocked - - `bulk`: matching traffic will be moved to low priority QoS class named `Bulk` - - `best_effort`: matching traffic will be moved to average priority QoS class named `Best Effort` - - `video`: matching traffic will be moved to high priority QoS class named `Video` - - `voice`: matching traffic will be moved to very high priority QoS class named `Voice` -- `description`: an optional rule description -- `device`: optional device name, if set the rule will be applied only to the given device, example `br-lan` -- `application`: list of applications to match, the list can contain application names like `netify.amazon-prime` -- `enabled`: can be `0` or `1`, if set to `1` the rule will be enabled -- `log`: can be set to `1` to log matching connections, gives an improved visibility on matched connections; - logs are stored in `/var/run/netifyd/dpi-actions-*.json` - -Global exemptions options: - -- `criteria`: global exemption criteria, usually it's an IP address; it can also be an object like `/` -- `enabled`: can be `0` or `1`, if set to `1` enable the exemption -- `description`: an optional exemption description - -All enabled rules are always evaluated by netifyd, the rule order doesn't matter. - -Example of `/etc/config/dpi`: -``` -config main 'config' - option log_blocked '1' - option firewall_exemption '1' - list popular_filters 'netify.netflix' - list popular_filters 'netify.telegram' - list popular_filters 'DoT' - list popular_filters 'netify.twitch' - list popular_filters 'netify.teamviewer' - list popular_filters 'DoH' - -config rule - option action 'bulk' - option criteria 'local_ip == 192.168.100.22 && application == "netify.facebook";' - option description 'Low priority for 192.168.100.22 when accessing Facebook' - option enabled 1 - -config rule 'ns_e775b8a7' - option enabled '1' - option device 'br-lan' - option action 'block' - list application 'netify.amazon-prime' - option description 'Block Amazon Prime for everyone' +## Configuration -config rule - option action 'block' - list application 'netify.twitter - list application 'netify.instagram' - list source 'objects/ns_hostset_1' - list source 'dhcp/ns_reservation_1' - option description 'Block Twitter and Instagrm for some hosts' - option enabled 1 - -config exemption - option criteria '192.168.1.22' - option description 'Important host' - option enabled '1' +Global options, in the `config` section: -config exemption - option criteria 'dhcp/ns_271ca281' - option description 'Important host with a reservation' - option enabled '1' -``` +- `log_blocked`: `1` logs the blocked connections in `/var/log/messages`, with the `DPI block: ` prefix -QoS rules do not have any effect if qosify is not enabled. -To enable qosify use: -``` -uci set qosify.wan.disabled=0 -uci commit qosify -/etc/init.d/qosify restart -``` +Application group (`appgroup` section), a named set of members: -To inspect qosify status use: -``` -qosify-status -``` +- `ns_name`: name of the group +- `app`: list of application names, as reported by the engine (e.g. `netify.netflix`) +- `app_category`: list of application category tags (e.g. `games`) +- `proto`: list of protocol names, as reported by the engine (e.g. `HTTP/Connect`) +- `proto_category`: list of protocol category tags -Check if traffic is matching: -``` -nft list table inet dpi -``` +Rule (`rule` section): -List connections with `block` labels: -``` -conntrack -L -o label -l block -``` +- `ns_name`: name of the rule +- `enabled`: `0` or `1` +- `action`: `block` or `allow` +- `priority`: evaluation order, starting at `1`; the first rule matching a flow wins +- `appgroup`: list of application groups to match +- `source`: list of addresses, networks or ranges to narrow the rule to; empty means every host +- `ns_match_all`: `1` matches every flow instead of naming application groups +- `ns_managed`: `1` for the rules created by the API +- `criteria`: raw netifyd expression, used by the rules not created by the API; the API can't edit them -If needed, start netifyd in debug mode: -``` -netifyd -R -d -I br-lan -E eth1 +Example: ``` +config main 'config' + option log_blocked '1' -## Supplementary signatures - -By default, netifyd is equipped to detect around 430 protocols and applications. With the inclusion of -supplementary signatures, netifyd can extend its recognition capabilities to encompass over 1600 protocols and applications. - -Extra signatures are accessible only from a machine with a valid subscription. -A cron job will update DPI signatures during the night and upon machine registration. -The download will be authenticated using a Nethesis proxy. +config appgroup 'ns_1a2b3c4d' + option ns_name 'Streaming and games' + list app 'netify.netflix' + list app_category 'games' -To force the update execute: -``` -dpi-update +config rule 'ns_3869dc35' + option ns_name 'Allow the office' + option ns_managed '1' + option enabled '1' + option action 'allow' + option priority '1' + option ns_match_all '1' + list source '192.168.1.10' + +config rule 'ns_9d40be71' + option ns_name 'Block streaming and games' + option ns_managed '1' + option enabled '1' + option action 'block' + option priority '2' + list appgroup 'ns_1a2b3c4d' + list source '192.168.1.0/24' ``` -You can use a different proxy by overriding the `HOST` env variable. -If the proxy is authenticated, use the `__USER__` and `__PASSWORD__` placeholders. -The placeholders will be replaced with systemd id and secret from `ns-plug`. +## Migration from the previous schema -Example: -``` -HOST=http://__USER__:__PASSWORD__@sp.gs.nethserver.net dpi-update -``` +On upgrade, and every time the DPI service starts, `/usr/libexec/ns-dpi/dpi-migrate` converts the +configuration written before application groups existed, e.g. after restoring an old backup: -## Managing Interface Exclusions in Netifyd +- every rule with `device`, `application`, `protocol` or `category` becomes an unmanaged rule, with its + behaviour kept in `criteria`. Rules with an action other than `block`, such as the QoS ones (`bulk`, + `best_effort`, `video`, `voice`), and rules matching nothing are dropped. The per-rule `exemption` and + `log` options are removed +- every `exemption` section becomes an `allow` rule at the top of the list, named after the exemption + description +- `popular_filters` is removed -By default, Netifyd monitors all interfaces. To exclude specific interfaces, you can define an exclusion list. Below are commands to add, modify, or remove excluded interfaces. +Dropped rules and removed options are logged in `/var/log/messages`. The `enabled` and `firewall_exemption` +global options, which no longer switch anything, are removed on upgrade as well. -- Add interfaces to exclusion list -``` -uci add_list netifyd.@netifyd[0].ns_exclude='eth1' -uci add_list netifyd.@netifyd[0].ns_exclude='tun*' -uci add_list netifyd.@netifyd[0].ns_exclude='wg*' -uci commit netifyd -echo '{"changes": {"network": {}}}' | /usr/libexec/rpcd/ns.commit call commit -``` +## Troubleshooting -- Modify exclusion list +Check the generated rules: ``` -uci delete netifyd.@netifyd[0].ns_exclude='eth1' -uci add_list netifyd.@netifyd[0].ns_exclude='eth2' -uci commit netifyd -echo '{"changes": {"network": {}}}' | /usr/libexec/rpcd/ns.commit call commit +cat /etc/netifyd/netify-proc-flow-actions.json +nft list chain inet fw4 dpi_actions ``` -- Clear exclusion list +List the blocked connections: ``` -uci delete netifyd.@netifyd[0].ns_exclude -uci commit netifyd -echo '{"changes": {"network": {}}}' | /usr/libexec/rpcd/ns.commit call commit +conntrack -L -o label -l netify-blocked ``` -- Return the exclusion list -``` -uci show netifyd.@netifyd[0].ns_exclude -``` +## Signatures and catalogs + +- `dpi-update` downloads the extra signatures, available only with a valid subscription, every night; + run it by hand to force an update. On unregistration the extra signatures are removed and the ones + shipped with the image are restored +- `dpi-data-update` downloads the application and protocol catalogs (labels, categories, icons) into + `/etc/netifyd`, every night diff --git a/packages/ns-dpi/files/20_dpi b/packages/ns-dpi/files/20_dpi index 662d78a76..3db4b3df6 100644 --- a/packages/ns-dpi/files/20_dpi +++ b/packages/ns-dpi/files/20_dpi @@ -1,10 +1,14 @@ #!/bin/sh +# TODO: ship /etc/config/dpi as a conffile instead of creating it here. Before doing so: +# - 19-ns-api.wizard detects a fresh install by the absence of /etc/config/dpi, it needs another marker +# - the dpi-update cron entry below must move to its own uci-default +# - fix the conffiles define in the Makefile, it is declared for ns-plug instead of ns-dpi + if ! uci -q get dpi.config > /dev/null; then uci -q import dpi << EOI config main 'config' option log_blocked '0' - option firewall_exemption '1' EOI fi diff --git a/packages/ns-dpi/files/22_dpi_firewall_exemption b/packages/ns-dpi/files/22_dpi_firewall_exemption deleted file mode 100755 index 15463c920..000000000 --- a/packages/ns-dpi/files/22_dpi_firewall_exemption +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# -# Copyright (C) 2026 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Enable firewall_exemption on systems installed before it became the default, so DPI rules can not lock -# the administrator out of the unit. - -if [ "$(uci -q get dpi.config.firewall_exemption)" != "1" ]; then - uci -q set dpi.config.firewall_exemption='1' - uci commit dpi -fi diff --git a/packages/ns-dpi/files/23_dpi_drop_enabled b/packages/ns-dpi/files/23_dpi_drop_enabled deleted file mode 100755 index f1e002ab9..000000000 --- a/packages/ns-dpi/files/23_dpi_drop_enabled +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# -# Copyright (C) 2026 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Remove the global enabled flag from systems installed before netifyd ran all the time: it no longer -# switches anything, DPI filters traffic as soon as one rule is enabled. - -if uci -q get dpi.config.enabled > /dev/null; then - uci -q delete dpi.config.enabled - uci commit dpi -fi diff --git a/packages/ns-dpi/files/23_dpi_drop_options b/packages/ns-dpi/files/23_dpi_drop_options new file mode 100755 index 000000000..db1281a98 --- /dev/null +++ b/packages/ns-dpi/files/23_dpi_drop_options @@ -0,0 +1,21 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Remove the global options that no longer switch anything: +# - enabled: netifyd runs all the time, DPI filters traffic as soon as one rule is enabled +# - firewall_exemption: the firewall IP addresses are always excluded from the DPI rules + +changed=0 +for option in enabled firewall_exemption; do + if uci -q get dpi.config.$option > /dev/null; then + uci -q delete dpi.config.$option + changed=1 + fi +done + +[ "$changed" = "1" ] && uci commit dpi +exit 0 diff --git a/packages/ns-dpi/files/dpi b/packages/ns-dpi/files/dpi index 267e6d6fe..2b25982cf 100755 --- a/packages/ns-dpi/files/dpi +++ b/packages/ns-dpi/files/dpi @@ -5,8 +5,14 @@ # SPDX-License-Identifier: GPL-2.0-only # +# Shared with the iface hotplug, so the migration and the config generation never run twice at once +lock /var/lock/dpi.lock +# Convert a configuration still on the previous schema, e.g. restored from an old backup or synced +# by an HA peer: a no-op when there is nothing to migrate +/usr/libexec/ns-dpi/dpi-migrate # Generate netify flow actions config file /usr/sbin/dpi-config +lock -u /var/lock/dpi.lock # Apply nftables rules /usr/sbin/dpi-nft # reload netifyd daemon diff --git a/packages/ns-dpi/files/dpi-config b/packages/ns-dpi/files/dpi-config index ee9df2268..10a60a7da 100755 --- a/packages/ns-dpi/files/dpi-config +++ b/packages/ns-dpi/files/dpi-config @@ -10,7 +10,7 @@ import json import subprocess import os import syslog -from nethsec import utils, objects, dpi +from nethsec import dpi def get_interface_ips(interface): ret = list() @@ -44,13 +44,6 @@ config = { "netify-blocked" ] }, - "log": { - "target_type": "log", - "target_enabled": True, - "interval": 60, - "path": "/var/run/netifyd/", - "prefix": "dpi-actions", - }, "allow": { "target_type": "ctlabel", "target_enabled": True, @@ -81,30 +74,16 @@ config = { "exemptions": [] } -# Add firewall IP address as global exemptions +# Add firewall IP address as global exemptions, so DPI rules can not lock the administrator out of the unit u = EUci() -if u.get('dpi', 'config', 'firewall_exemption', default=0) == '1': - for section in u.get_all('network'): - if u.get('network', section) == 'interface': - ip_addrs = get_interface_ips(section) - if ip_addrs: - config["exemptions"] = config["exemptions"] + ip_addrs +for section in u.get_all('network'): + if u.get('network', section) == 'interface': + ip_addrs = get_interface_ips(section) + if ip_addrs: + config["exemptions"] = config["exemptions"] + ip_addrs -# Add custom global exemptions -exemptions = utils.get_all_by_type(u, 'dpi', 'exemption') -for ex_name in exemptions: - exemption = exemptions[ex_name] - criteria = exemption.get('criteria', '') - enabled = exemption.get('enabled', '1') - if enabled == '1' and criteria: - if objects.is_object_id(criteria): - for ip in objects.get_object_ips(u, criteria): - config["exemptions"].append(ip) - else: - config["exemptions"].append(criteria) - -rcount = 0 -valid_actions = ['block', 'bulk', 'best_effort', 'video', 'voice'] +# a rule from the schema used before application groups existed carries at least one of these fields; +# dpi.migrate_schema converts it, so one still here was not migrated yet legacy_fields = ('device', 'application', 'protocol', 'category') for section in u.get_all('dpi'): if u.get('dpi', section) != 'rule': @@ -113,76 +92,42 @@ for section in u.get_all('dpi'): if not rule.get('enabled'): continue - # a rule from the schema used before application groups existed carries at least one of these - # fields; their presence is a permanent marker, unrelated to whether the migration has since given - # the section a priority - is_legacy = any(field in rule for field in legacy_fields) - - if not is_legacy and 'criteria' not in rule: - # current schema: application groups, optionally narrowed to a source, or a match-all rule - if rule.get('action') not in dpi.DPI_RULE_ACTIONS: - syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, unknown action {rule.get('action')}") - continue - criteria = dpi.build_rule_criteria(u, rule) - if not criteria: - # never emit an empty criteria: it is a parse error, and a parse error kills the agent - syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, it matches nothing") - continue - try: - priority = int(rule.get('priority', 1)) - except ValueError: - syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, invalid priority") - continue - config["actions"][section] = { - "enabled": rule['enabled'] == '1', - "priority": priority, - "halt_on_match": True, - "criteria": criteria, - "targets": [rule['action']], - "exemptions": [] - } + if any(field in rule for field in legacy_fields): + # never emit it as a current rule: its source alone would block every flow of those hosts + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, not migrated to the current schema") continue - - if not is_legacy: - # raw criteria: hand-written, or a rule migrated from the old schema with its behaviour frozen - # verbatim (dpi.migrate_schema) — halts on match and keeps its own priority, same as any other - # rule of the current schema - if rule.get('action') not in dpi.DPI_RULE_ACTIONS: - syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, unknown action {rule.get('action')}") - continue - try: - priority = int(rule.get('priority', 1)) - except ValueError: - syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, invalid priority") - continue - config["actions"][section] = { - "enabled": rule['enabled'] == '1', - "priority": priority, - "halt_on_match": True, - "criteria": dpi.freeze_legacy_criteria(u, rule), - "targets": [rule['action']], - "exemptions": [] - } + if rule.get('action') not in dpi.DPI_RULE_ACTIONS: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, unknown action {rule.get('action')}") + continue + try: + priority = int(rule.get('priority', 1)) + except ValueError: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, invalid priority") continue - # legacy rule, still on the old schema: emitted as it was until the migration converts it - if rule.get('action') not in valid_actions: + if 'criteria' in rule: + # raw criteria: hand-written, or migrated from the old schema with its behaviour frozen + # (dpi.migrate_schema). Double quotes become single ones, and the terminator is added when + # missing: an unterminated expression is a syntax error + criteria = rule['criteria'].replace('"', "'").strip() + if criteria and not criteria.endswith(';'): + criteria = f'{criteria};' + else: + # application groups, optionally narrowed to a source, or a match-all rule + criteria = dpi.build_rule_criteria(u, rule) + if not criteria: + # never emit an empty criteria: it is a parse error, and a parse error kills the agent + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, it matches nothing") continue - targets = [rule['action']] - if 'log' in rule and rule['log'] == '1': - targets.append('log') - config["actions"][f"rule{rcount}"] = { + config["actions"][section] = { "enabled": rule['enabled'] == '1', - # legacy rules never halt, so they are evaluated after the ones of the current schema and - # their relative order does not matter - "priority": 1000 + rcount, - "halt_on_match": False, - "criteria": dpi.freeze_legacy_criteria(u, rule), - "targets": targets, - "exemptions": rule.get('exemption', []) + "priority": priority, + "halt_on_match": True, + "criteria": criteria, + "targets": [rule['action']], + "exemptions": [] } - rcount = rcount + 1 # if directories do not exist, create them if not os.path.exists(os.path.dirname(cfg_file)): diff --git a/packages/ns-dpi/files/dpi-exemption.hotplug b/packages/ns-dpi/files/dpi-exemption.hotplug index a1a60a86f..64442acc3 100755 --- a/packages/ns-dpi/files/dpi-exemption.hotplug +++ b/packages/ns-dpi/files/dpi-exemption.hotplug @@ -5,19 +5,19 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Keep the firewall_exemption list in sync with interface addresses that change at runtime (e.g. DHCP or -# PPPoE on WAN). Only netifyd is reloaded, never restarted, and only when the generated config changed. +# Keep the firewall IP addresses excluded from DPI in sync with interface addresses that change at runtime +# (e.g. DHCP or PPPoE on WAN). Only netifyd is reloaded, never restarted, and only when the generated config +# changed. [ "$ACTION" = "ifup" ] || { [ "$ACTION" = "ifupdate" ] && [ "$IFUPDATE_ADDRESSES" = "1" ]; } || exit 0 -[ "$(uci -q get dpi.config.firewall_exemption)" = "1" ] || exit 0 cfg_file=/etc/netifyd/netify-proc-flow-actions.json -lock /var/lock/dpi-exemption.lock +lock /var/lock/dpi.lock before=$(md5sum "$cfg_file" 2>/dev/null) /usr/sbin/dpi-config after=$(md5sum "$cfg_file" 2>/dev/null) -lock -u /var/lock/dpi-exemption.lock +lock -u /var/lock/dpi.lock # a stopped agent (e.g. HA backup node) reads the updated file when it starts if [ "$before" != "$after" ] && /etc/init.d/netifyd running; then diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py index e6b16b6b4..7c640ca98 100644 --- a/packages/python3-nethsec/src/nethsec/dpi/__init__.py +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -956,8 +956,7 @@ def freeze_legacy_criteria(e_uci: EUci, rule: dict) -> str: Build the criteria of a rule written before application groups existed, the way `dpi-config` has always generated it: source, application, protocol and category turned into an expression narrowed to the rule's device, with the VLAN rewrite applied when the device names a VLAN. Used by the - generator for a legacy rule the migration has not converted yet, and by the migration itself to - freeze a rule's behaviour verbatim before clearing those fields. + migration to freeze a rule's behaviour verbatim before clearing those fields. Args: - e_uci: euci instance @@ -1037,12 +1036,15 @@ def migrate_schema(e_uci: EUci): Each legacy rule (`device`, `application`, `protocol` or `category` set) is turned into an unmanaged rule with its behaviour frozen verbatim into `criteria` — see `freeze_legacy_criteria`. A rule whose - action is a retired QoS value, carries a per-rule exemption (a field no recent API ever wrote), or - ends up matching nothing is dropped instead, with a log line naming it. + action is a retired QoS value, or that ends up matching nothing, is dropped instead, with a log line + naming it. A per-rule exemption or `log` (fields no recent API ever wrote) is removed from the rule, + with a log line as well. Each global exemption becomes an Allow rule at the top of the list: a managed match-all rule narrowed to a source when its criteria is a plain address, CIDR or firewall object; an unmanaged - one carrying the criteria verbatim otherwise. Disabled exemptions become disabled rules. + one carrying the criteria verbatim otherwise. Disabled exemptions become disabled rules. The rule + is named after the exemption description, cut to the rule name limit, or `Migrated exception N` + when it has none. The `exemption` section type and `popular_filters` are then removed. The changes are committed and the dpi service reloaded right away. @@ -1067,9 +1069,12 @@ def migrate_schema(e_uci: EUci): exemption_count = 0 for section, exemption in exemptions.items(): exemption_count += 1 - name = f'Migrated exception {exemption_count}' + # keep the name the user gave the exemption, cut to what a rule name allows + name = exemption.get('description', '').strip()[:DPI_RULE_NAME_MAX_LENGTH].strip() + if not name: + name = f'Migrated exception {exemption_count}' enabled = exemption.get('enabled', '1') == '1' - criteria = (exemption.get('criteria') or '').strip() + criteria = exemption.get('criteria', '').strip() source = __exemption_source(e_uci, criteria) e_uci.delete('dpi', section) @@ -1098,6 +1103,9 @@ def migrate_schema(e_uci: EUci): if rule.get('exemption'): syslog.syslog(syslog.LOG_WARNING, f"dpi migration: dropping the per-rule exemption on {section}, no longer supported") + if rule.get('log') == '1': + syslog.syslog(syslog.LOG_WARNING, + f"dpi migration: dropping the per-rule log on {section}, no longer supported") if not any(rule.get(field) for field in ('criteria', 'source', 'application', 'protocol', 'category')): syslog.syslog(syslog.LOG_WARNING, f"dpi migration: dropping {section}, it matches nothing") e_uci.delete('dpi', section) @@ -1105,7 +1113,7 @@ def migrate_schema(e_uci: EUci): rule_count += 1 criteria = freeze_legacy_criteria(e_uci, rule) - for field in (*legacy_fields, 'source', 'exemption'): + for field in (*legacy_fields, 'source', 'exemption', 'log'): e_uci.delete('dpi', section, field) e_uci.set('dpi', section, 'criteria', criteria) e_uci.set('dpi', section, 'ns_name', f'Migrated rule {rule_count}') diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py index ff5d5c72c..739585915 100644 --- a/packages/python3-nethsec/tests/test_dpi.py +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -55,13 +55,11 @@ dpi_minimal_db = """ config main 'config' option log_blocked '0' - option firewall_exemption '0' """ dpi_db = """ config main 'config' option log_blocked '0' - option firewall_exemption '0' list popular_filters 'netify.netflix' list popular_filters 'netify.hulu' list popular_filters 'netify.whatsapp' @@ -1045,7 +1043,6 @@ def test_freeze_legacy_criteria_applies_vlan_rewrite(e_uci: EUci): legacy_dpi_db = """ config main 'config' option log_blocked '0' - option firewall_exemption '1' list popular_filters 'netify.facebook' config rule 'ns_b01a0e73' @@ -1146,10 +1143,31 @@ def test_migrate_schema_drops_per_rule_exemption_but_keeps_the_rule(e_uci_legacy assert 'criteria' in rule +def test_migrate_schema_drops_per_rule_log_but_keeps_the_rule(e_uci, mocker): + syslog = mocker.patch('nethsec.dpi.syslog.syslog') + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(""" +config main 'config' + +config rule 'ns_logged' + option enabled '1' + option device 'eth0' + option action 'block' + option log '1' + list application 'netify.instagram' +""") + dpi.migrate_schema(e_uci) + rule = e_uci.get_all('dpi', 'ns_logged') + assert 'log' not in rule + assert rule['action'] == 'block' + assert 'criteria' in rule + assert any('per-rule log on ns_logged' in call.args[1] for call in syslog.call_args_list) + + def test_migrate_schema_converts_address_exemption_to_managed_allow_rule(e_uci_legacy): dpi.migrate_schema(e_uci_legacy) rules = dpi.list_rules(e_uci_legacy) - exception1 = next(r for r in rules if r['name'] == 'Migrated exception 1') + exception1 = next(r for r in rules if r['name'] == 'my exception') assert exception1['managed'] is True assert exception1['action'] == 'allow' assert exception1['source'] == ['192.168.122.47'] @@ -1161,13 +1179,14 @@ def test_migrate_schema_converts_address_exemption_to_managed_allow_rule(e_uci_l def test_migrate_schema_keeps_a_disabled_exemption_disabled(e_uci_legacy): dpi.migrate_schema(e_uci_legacy) rules = dpi.list_rules(e_uci_legacy) - exception2 = next(r for r in rules if r['name'] == 'Migrated exception 2') + exception2 = next(r for r in rules if r['name'] == 'disabled one') assert exception2['enabled'] is False def test_migrate_schema_converts_non_address_exemption_to_unmanaged_allow_rule(e_uci_legacy): dpi.migrate_schema(e_uci_legacy) rules = dpi.list_rules(e_uci_legacy) + # no description: the name falls back to its position exception3 = next(r for r in rules if r['name'] == 'Migrated exception 3') assert exception3['managed'] is False assert exception3['action'] == 'allow' @@ -1193,11 +1212,26 @@ def test_migrate_schema_converts_object_exemption_via_expanded_addresses(e_uci, assert rules[0]['source'] == ['192.168.50.0/24'] +def test_migrate_schema_cuts_a_long_exemption_description_to_the_rule_name_limit(e_uci): + description = 'x' * (dpi.DPI_RULE_NAME_MAX_LENGTH + 10) + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(f""" +config main 'config' + +config exemption 'ns_long' + option enabled '1' + option criteria '192.168.1.1' + option description '{description}' +""") + dpi.migrate_schema(e_uci) + rules = dpi.list_rules(e_uci) + assert rules[0]['name'] == description[:dpi.DPI_RULE_NAME_MAX_LENGTH] + assert rules[0]['managed'] is True + + def test_migrate_schema_removes_exemption_sections_and_legacy_globals(e_uci_legacy): dpi.migrate_schema(e_uci_legacy) assert utils.get_all_by_type(e_uci_legacy, 'dpi', 'exemption') == {} - # still read by dpi-config and the hotplug script, so the migration must leave it alone - assert e_uci_legacy.get('dpi', 'config', 'firewall_exemption') == '1' assert e_uci_legacy.get('dpi', 'config', 'popular_filters', list=True, default=None) is None @@ -1205,7 +1239,7 @@ def test_migrate_schema_puts_exemptions_before_migrated_rules(e_uci_legacy): dpi.migrate_schema(e_uci_legacy) rules = dpi.list_rules(e_uci_legacy) names_in_order = [r['name'] for r in rules] - assert names_in_order.index('Migrated exception 1') < names_in_order.index('Migrated rule 1') + assert names_in_order.index('my exception') < names_in_order.index('Migrated rule 1') def test_migrate_schema_applies_immediately(e_uci_legacy, mock_reload): diff --git a/packages/python3-nethsec/tests/test_inventory.py b/packages/python3-nethsec/tests/test_inventory.py index ca151c811..6127e7807 100644 --- a/packages/python3-nethsec/tests/test_inventory.py +++ b/packages/python3-nethsec/tests/test_inventory.py @@ -561,7 +561,6 @@ dpi_db = """ config main 'config' option log_blocked '0' - option firewall_exemption '0' config rule 'ns_2b170d05' option enabled '1' From ac4404213e1c811aed428a46b5b2252272ccd1b5 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Mon, 5 Oct 2026 09:45:33 +0200 Subject: [PATCH 14/17] feat(dpi): update UI source version to specific commit v7 --- packages/ns-ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 90b315335..bc36406b0 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=588121f04ffbd1c7df7f9959f005578afc75a415 +PKG_SOURCE_VERSION:=f8ae66ffc4b141c38b83ab0f914f02a41fdc0b9b PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip From cc5aa98b152c2c9c00ec0e645bf93c71d2946f16 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Mon, 5 Oct 2026 11:44:10 +0200 Subject: [PATCH 15/17] refactor(dpi): remove hotplug exemption script and update documentation for wan interface exclusions --- packages/ns-dpi/Makefile | 2 -- packages/ns-dpi/README.md | 5 ++-- packages/ns-dpi/files/23_dpi_drop_options | 2 +- packages/ns-dpi/files/dpi | 3 --- packages/ns-dpi/files/dpi-config | 28 +++------------------ packages/ns-dpi/files/dpi-exemption.hotplug | 25 ------------------ 6 files changed, 7 insertions(+), 58 deletions(-) delete mode 100755 packages/ns-dpi/files/dpi-exemption.hotplug diff --git a/packages/ns-dpi/Makefile b/packages/ns-dpi/Makefile index 59ccb8567..134747449 100644 --- a/packages/ns-dpi/Makefile +++ b/packages/ns-dpi/Makefile @@ -67,7 +67,6 @@ define Package/ns-dpi/install $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/unregister $(INSTALL_DIR) $(1)/etc/xtables - $(INSTALL_DIR) $(1)/etc/hotplug.d/iface $(INSTALL_CONF) ./files/connlabel.conf $(1)/etc/connlabel.conf $(LN) /etc/connlabel.conf $(1)/etc/xtables/connlabel.conf $(INSTALL_BIN) ./files/dpi.init $(1)/etc/init.d/dpi @@ -88,7 +87,6 @@ define Package/ns-dpi/install $(INSTALL_BIN) ./files/99-dpi-data-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-data-update-cron $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/register $(INSTALL_BIN) ./files/70dpi $(1)/usr/share/ns-plug/hooks/unregister/ - $(INSTALL_BIN) ./files/dpi-exemption.hotplug $(1)/etc/hotplug.d/iface/90-dpi-exemption endef $(eval $(call BuildPackage,ns-dpi)) diff --git a/packages/ns-dpi/README.md b/packages/ns-dpi/README.md index fc308a832..e5bbbe5ae 100644 --- a/packages/ns-dpi/README.md +++ b/packages/ns-dpi/README.md @@ -10,9 +10,8 @@ How it works: flows labelled `netify-blocked` netifyd runs all the time: traffic is filtered as soon as one rule is enabled. -The firewall IP addresses are always excluded from the DPI rules, so a rule can not block access to the -firewall itself; addresses changing at runtime (DHCP, PPPoE) are tracked by -`/etc/hotplug.d/iface/90-dpi-exemption`. +Flows seen on the WAN interfaces are excluded from the DPI rules (global `iface == 'wan'` exemption, as +in the netifyd `10-nfqueue.conf`). Rules and application groups are managed by the `ns.dpi` API, see `packages/ns-api/README.md`. After editing `/etc/config/dpi` by hand, apply the changes with: ``` diff --git a/packages/ns-dpi/files/23_dpi_drop_options b/packages/ns-dpi/files/23_dpi_drop_options index db1281a98..fa23094c3 100755 --- a/packages/ns-dpi/files/23_dpi_drop_options +++ b/packages/ns-dpi/files/23_dpi_drop_options @@ -7,7 +7,7 @@ # Remove the global options that no longer switch anything: # - enabled: netifyd runs all the time, DPI filters traffic as soon as one rule is enabled -# - firewall_exemption: the firewall IP addresses are always excluded from the DPI rules +# - firewall_exemption: the flows seen on the WAN interfaces are always excluded from the DPI rules changed=0 for option in enabled firewall_exemption; do diff --git a/packages/ns-dpi/files/dpi b/packages/ns-dpi/files/dpi index 2b25982cf..228d33859 100755 --- a/packages/ns-dpi/files/dpi +++ b/packages/ns-dpi/files/dpi @@ -5,14 +5,11 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Shared with the iface hotplug, so the migration and the config generation never run twice at once -lock /var/lock/dpi.lock # Convert a configuration still on the previous schema, e.g. restored from an old backup or synced # by an HA peer: a no-op when there is nothing to migrate /usr/libexec/ns-dpi/dpi-migrate # Generate netify flow actions config file /usr/sbin/dpi-config -lock -u /var/lock/dpi.lock # Apply nftables rules /usr/sbin/dpi-nft # reload netifyd daemon diff --git a/packages/ns-dpi/files/dpi-config b/packages/ns-dpi/files/dpi-config index 10a60a7da..fa3748b29 100755 --- a/packages/ns-dpi/files/dpi-config +++ b/packages/ns-dpi/files/dpi-config @@ -7,27 +7,10 @@ from euci import EUci import json -import subprocess import os import syslog from nethsec import dpi -def get_interface_ips(interface): - ret = list() - try: - proc = subprocess.run(['/sbin/ifstatus', interface], check=True, capture_output=True) - info = json.loads(proc.stdout) - except: - return ret - - for ip_type in ('ipv4-address', 'ipv6-address'): - if ip_type in info: - for addr in info[ip_type]: - if 'address' in addr: - ret.append(addr['address']) - - return ret - cfg_file = "/etc/netifyd/netify-proc-flow-actions.json" config = { "reprocess_flows": True, @@ -71,16 +54,13 @@ config = { "exemptions": [] } }, - "exemptions": [] + # based on 10-nfqueue.conf in netifyd + "exemptions": [ + "iface == 'wan';" + ] } -# Add firewall IP address as global exemptions, so DPI rules can not lock the administrator out of the unit u = EUci() -for section in u.get_all('network'): - if u.get('network', section) == 'interface': - ip_addrs = get_interface_ips(section) - if ip_addrs: - config["exemptions"] = config["exemptions"] + ip_addrs # a rule from the schema used before application groups existed carries at least one of these fields; # dpi.migrate_schema converts it, so one still here was not migrated yet diff --git a/packages/ns-dpi/files/dpi-exemption.hotplug b/packages/ns-dpi/files/dpi-exemption.hotplug deleted file mode 100755 index 64442acc3..000000000 --- a/packages/ns-dpi/files/dpi-exemption.hotplug +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# -# Copyright (C) 2026 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -# Keep the firewall IP addresses excluded from DPI in sync with interface addresses that change at runtime -# (e.g. DHCP or PPPoE on WAN). Only netifyd is reloaded, never restarted, and only when the generated config -# changed. - -[ "$ACTION" = "ifup" ] || { [ "$ACTION" = "ifupdate" ] && [ "$IFUPDATE_ADDRESSES" = "1" ]; } || exit 0 - -cfg_file=/etc/netifyd/netify-proc-flow-actions.json - -lock /var/lock/dpi.lock -before=$(md5sum "$cfg_file" 2>/dev/null) -/usr/sbin/dpi-config -after=$(md5sum "$cfg_file" 2>/dev/null) -lock -u /var/lock/dpi.lock - -# a stopped agent (e.g. HA backup node) reads the updated file when it starts -if [ "$before" != "$after" ] && /etc/init.d/netifyd running; then - /etc/init.d/netifyd reload -fi From 12b3f7ca141c5be9670582251c45e49a938de740 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 8 Oct 2026 16:31:26 +0200 Subject: [PATCH 16/17] added metrics to analyze usage --- .../src/nethsec/inventory/__init__.py | 42 ++++++++++++++-- .../python3-nethsec/tests/test_inventory.py | 48 +++++++++++++++---- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/packages/python3-nethsec/src/nethsec/inventory/__init__.py b/packages/python3-nethsec/src/nethsec/inventory/__init__.py index ae370128f..cf22273a7 100644 --- a/packages/python3-nethsec/src/nethsec/inventory/__init__.py +++ b/packages/python3-nethsec/src/nethsec/inventory/__init__.py @@ -641,11 +641,43 @@ def fact_proxy_pass(uci: EUci): return ret def fact_dpi(uci: EUci): - ret = {"enabled": False, "rules": 0} - for rule in utils.get_all_by_type(uci, 'dpi', 'rule'): - if uci.get('dpi', rule, 'enabled', default='0') == '1': - ret["rules"] += 1 - ret["enabled"] = ret["rules"] > 0 + ret = { + "total": 0, + "enabled": 0, + "action": {"block": 0, "allow": 0}, + "source": {"all": 0, "ip": 0}, + "match": {"all": 0, "appgroup": 0}, + "kind": {"managed": 0, "unmanaged": 0}, + "appgroups": {"total": 0, "used": 0, "app": 0, "app_category": 0, "proto": 0, "proto_category": 0}, + } + used_appgroups = set() + for rule in (utils.get_all_by_type(uci, 'dpi', 'rule') or {}).values(): + ret["total"] += 1 + if rule.get('enabled', '0') == '1': + ret["enabled"] += 1 + if rule.get('action') in ret["action"]: + ret["action"][rule.get('action')] += 1 + # raw criteria is emitted as is, source and appgroup don't apply + if not rule.get('criteria'): + if rule.get('source'): + ret["source"]["ip"] += 1 + else: + ret["source"]["all"] += 1 + if rule.get('ns_match_all', '0') == '1': + ret["match"]["all"] += 1 + elif rule.get('appgroup'): + ret["match"]["appgroup"] += 1 + if rule.get('ns_managed', '0') == '1': + ret["kind"]["managed"] += 1 + else: + ret["kind"]["unmanaged"] += 1 + used_appgroups.update(rule.get('appgroup', [])) + for name, appgroup in (utils.get_all_by_type(uci, 'dpi', 'appgroup') or {}).items(): + ret["appgroups"]["total"] += 1 + if name in used_appgroups: + ret["appgroups"]["used"] += 1 + for member in ('app', 'app_category', 'proto', 'proto_category'): + ret["appgroups"][member] += len(appgroup.get(member, [])) return ret def fact_extra_packages(uci: EUci): diff --git a/packages/python3-nethsec/tests/test_inventory.py b/packages/python3-nethsec/tests/test_inventory.py index 6127e7807..4654c5b17 100644 --- a/packages/python3-nethsec/tests/test_inventory.py +++ b/packages/python3-nethsec/tests/test_inventory.py @@ -562,11 +562,40 @@ config main 'config' option log_blocked '0' +config appgroup 'ns_1a2b3c4d' + option ns_name 'Streaming and games' + list app 'netify.netflix' + list app 'netify.youtube' + list app_category 'games' + +config appgroup 'ns_5e6f7a8b' + option ns_name 'Unused' + list proto 'HTTP/Connect' + list proto_category 'vpn' + +config rule 'ns_3869dc35' + option ns_name 'Allow the office' + option ns_managed '1' + option enabled '0' + option action 'allow' + option priority '1' + option ns_match_all '1' + list source '192.168.1.10' + config rule 'ns_2b170d05' + option ns_name 'Block streaming and games' + option ns_managed '1' option enabled '1' - option device 'br-lan' option action 'block' - list application 'netify.facebook' + option priority '2' + list appgroup 'ns_1a2b3c4d' + +config rule 'ns_9d40be71' + option ns_name 'Custom' + option enabled '1' + option action 'block' + option priority '3' + option criteria 'local_ip == 192.168.1.20;' """ dhcp_db = """ @@ -925,12 +954,15 @@ def test_fact_ipsec(tmp_path): def test_fact_dpi(tmp_path): u = _setup_db(tmp_path) - assert inventory.fact_dpi(u) == {"enabled": True, "rules": 1} - -def test_fact_dpi_without_enabled_rules(tmp_path): - u = _setup_db(tmp_path) - u.set('dpi', 'ns_2b170d05', 'enabled', '0') - assert inventory.fact_dpi(u) == {"enabled": False, "rules": 0} + assert inventory.fact_dpi(u) == { + "total": 3, + "enabled": 2, + "action": {"block": 2, "allow": 1}, + "source": {"all": 1, "ip": 1}, + "match": {"all": 1, "appgroup": 1}, + "kind": {"managed": 2, "unmanaged": 1}, + "appgroups": {"total": 2, "used": 1, "app": 2, "app_category": 1, "proto": 1, "proto_category": 1}, + } def test_fact_dhcp_server(tmp_path): u = _setup_db(tmp_path) From 21b4c03ebfe94dda72509c5bda74bcceb0ecfcdf Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Thu, 8 Oct 2026 16:31:42 +0200 Subject: [PATCH 17/17] adjusted ui version --- packages/ns-ui/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index bc36406b0..4d6ac49e0 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -7,12 +7,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-ui # renovate: datasource=github-releases depName=NethServer/nethsecurity-ui -PKG_VERSION:=2.26.1 +PKG_VERSION:=2.27.0 PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=f8ae66ffc4b141c38b83ab0f914f02a41fdc0b9b +PKG_SOURCE_VERSION:=$(PKG_VERSION) PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip