diff --git a/packages/banip/files/README.md b/packages/banip/files/README.md index 8109069b0..bbf5df08a 100644 --- a/packages/banip/files/README.md +++ b/packages/banip/files/README.md @@ -255,6 +255,7 @@ The `report` sub-command accepts an output mode: `text` (default, human-readable | ban_resolver | option | - | external resolver used for DNS lookups, by default the local resolver/forwarder will be used | | ban_remotelog | option | 0 | enable the cgi interface to receive remote logging events | | ban_remotetoken | option | - | unique token to communicate with the cgi interface | +| ban_blockhook | option | - | full path of an external script, called once for every IP newly blocked by the log service | ## Examples @@ -545,6 +546,18 @@ Examples to transfer remote logging events from an internal server to banIP via Please note: for security reasons use this cgi interface only internally and only encrypted via https transfer protocol. +**External hook for newly blocked IPs** +banIP can notify an external program whenever the log service adds a new IP to a blocklist Set (disabled by default). Set `ban_blockhook` to the full path of an executable script, e.g. `/usr/libexec/my-block-hook`. The hook is called once per newly blocked IP, right after the nftables element has been added, with four positional arguments: + +``` + $1: the blocked IP address, e.g. '198.51.100.44' + $2: the protocol family, either 'v4' or 'v6' + $3: the log count that triggered the block (ban_logcount) + $4: the Set expiry time (ban_nftexpiry), '0s' if the block is permanent +``` + +Please note: the hook runs synchronously inside the single-threaded log service, so it has to be fast and non-blocking - queue the event and process it elsewhere, never do network I/O in the hook itself. Its exit code and output are ignored. + **Download options** By default banIP uses the following pre-configured download options: diff --git a/packages/banip/files/banip-functions.sh b/packages/banip/files/banip-functions.sh index e8b5b54bd..e4fdb45d6 100755 --- a/packages/banip/files/banip-functions.sh +++ b/packages/banip/files/banip-functions.sh @@ -41,6 +41,7 @@ ban_mailprofile="ban_notify" ban_mailnotification="0" ban_remotelog="0" ban_remotetoken="" +ban_blockhook="" ban_nftloglevel="warn" ban_nftpriority="-100" ban_nftpolicy="memory" @@ -2835,6 +2836,12 @@ f_monitor() { fi block_cache="${block_cache} ${ip} " f_log "info" "add IP '${ip}' (cnt: ${ban_logcount}, expiry: ${ban_nftexpiry:-"0"}) to blocklist${proto} Set" + + # optional external hook, called once for every newly blocked IP + # + if [ -n "${ban_blockhook}" ] && [ -x "${ban_blockhook}" ]; then + "${ban_blockhook}" "${ip}" "${proto#.}" "${ban_logcount}" "${ban_nftexpiry:-0s}" >/dev/null 2>&1 + fi else f_log "info" "failed to add IP '${ip}' to blocklist${proto} Set with rc '${?}'" continue diff --git a/packages/ns-api/files/ns.threatshield b/packages/ns-api/files/ns.threatshield index a23678e78..716613c17 100644 --- a/packages/ns-api/files/ns.threatshield +++ b/packages/ns-api/files/ns.threatshield @@ -468,7 +468,7 @@ def list_blocklist(e_uci): feed = feeds[f] enabled = f in enabled_feeds - if 'nethesis-blacklists' in feed.get('url_4'): + if 'nethesis' in feed.get('url_4'): type = 'enterprise' else: type = 'community' diff --git a/packages/ns-threat_shield/Makefile b/packages/ns-threat_shield/Makefile index 29a0f9e67..57740c7ca 100644 --- a/packages/ns-threat_shield/Makefile +++ b/packages/ns-threat_shield/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-threat_shield -PKG_VERSION:=1.0.4 +PKG_VERSION:=1.1.0 PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/ns-threat_shield-$(PKG_VERSION) @@ -46,11 +46,13 @@ define Package/ns-threat_shield/install $(INSTALL_DIR) $(1)/usr/libexec $(INSTALL_BIN) ./files/ts-dns $(1)/usr/sbin/ts-dns $(INSTALL_BIN) ./files/ts-ip $(1)/usr/sbin/ts-ip + $(INSTALL_BIN) ./files/send-insights-blocklist $(1)/usr/sbin/send-insights-blocklist + $(INSTALL_BIN) ./files/ts-insights-hook $(1)/usr/libexec/ts-insights-hook $(INSTALL_BIN) ./files/20_threat_shield $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/ts-dns.hook $(1)/usr/share/ns-plug/hooks/register/90ts-dns $(INSTALL_BIN) ./files/ts-dns.hook $(1)/usr/share/ns-plug/hooks/unregister/90ts-dns - $(LN) /usr/sbin/ts-ip $(1)/usr/share/ns-plug/hooks/register/80ts-ip - $(LN) /usr/sbin/ts-ip $(1)/usr/share/ns-plug/hooks/unregister/80ts-ip + $(INSTALL_BIN) ./files/ts-ip.hook $(1)/usr/share/ns-plug/hooks/register/80ts-ip + $(INSTALL_BIN) ./files/ts-ip.hook $(1)/usr/share/ns-plug/hooks/unregister/80ts-ip $(INSTALL_DIR) $(1)/usr/share/threat_shield $(INSTALL_DATA) ./files/nethesis-dns.sources $(1)/usr/share/threat_shield $(INSTALL_DATA) ./files/community-dns.sources $(1)/usr/share/threat_shield @@ -62,6 +64,7 @@ define Package/ns-threat_shield/install $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/banip-defaults $(1)/etc/uci-defaults/99-nethsec-banip $(INSTALL_BIN) ./files/banip-extra-defaults $(1)/etc/uci-defaults/96-nethsec-banip-extra + $(INSTALL_BIN) ./files/banip-insights-defaults $(1)/etc/uci-defaults/97-nethsec-banip-insights $(INSTALL_BIN) ./files/35_ns-threat_shield $(1)/etc/uci-defaults/35_ns-threat_shield $(INSTALL_BIN) ./files/96_ns-threat_shield $(1)/etc/uci-defaults/96_ns-threat_shield gzip -9n $(1)/usr/share/threat_shield/nethesis-dns.sources @@ -84,6 +87,7 @@ define Package/ns-threat_shield/prerm if [ -z "$${IPKG_INSTROOT}" ]; then crontab -l | grep -v "/etc/init.d/banip reload" | sort | uniq | crontab - crontab -l | grep -v "/etc/init.d/adblock" | sort | uniq | crontab - + crontab -l | grep -v "/usr/sbin/send-insights-blocklist" | sort | uniq | crontab - fi exit 0 endef diff --git a/packages/ns-threat_shield/README.md b/packages/ns-threat_shield/README.md index 1773bbd5f..2fe48b9a4 100644 --- a/packages/ns-threat_shield/README.md +++ b/packages/ns-threat_shield/README.md @@ -21,6 +21,7 @@ The following categories require a valid entitlement: - `yoroisusplvl1` (was `yoroi_souspicious_level1` on NS7) - `yoroisusplvl2` (was `yoroi_souspicious_level2` on NS7) - `nethesislvl3` (was `nethesis_level3` on NS7) +- `nethesisinsightslvl2` (attackers reported by the other Nethesis firewalls, see [Nethesis Insights](#nethesis-insights)) After machine registration, above categories will be automatically added to existing banip categories (`/etc/banip/banip.custom.feeds`). @@ -47,6 +48,33 @@ ts-ip /etc/init.d/banip restart ``` +### Nethesis Insights + +On registered machines `ts-ip` joins the [Nethesis Insights](https://github.com/nethesis/nethesis-insights) +threat shield: IPs blocked by the banip log service are reported to the Insights server, and IPs reported by +several firewalls can be blocked locally with the `nethesisinsightslvl2` feed. +Authentication uses the subscription `system_id` and `secret`. + +| machine | allowlist | reports blocked IPs | `nethesisinsightslvl2` feed | +|---|---|---|---| +| no subscription | no | no | not available | +| subscription | yes | yes | not listed | +| subscription with the Threat Shield entitlement | yes | yes | selectable, disabled by default | + +Reporting: + +- `ts-ip` sets `banip.global.ban_blockhook` to `/usr/libexec/ts-insights-hook`, which appends every blocked IP + to `/var/run/ns-insights/threat-events.jsonl` +- banip supports a single block hook: when `ban_blockhook` already points to a custom script, `ts-ip` leaves it + untouched, Insights reporting stays disabled and a `threat_shield` message is logged to `/var/log/messages`; + on unregistration `ts-ip` removes only its own hook +- `/usr/sbin/send-insights-blocklist` sends the public IPs to `POST /blocklist/v1/events`, in batches of 500; + the full endpoint URL can be overridden with `ns-plug.config.insights_blocklist_url` + (default `https://insights.nethesis.it/blocklist/v1/events`); + `ts-ip` adds its 5-minute cron job on registration and removes it, along with the spool, on unregistration +- fire and forget: events that fail to send are dropped +- only failed pushes are logged to `/var/log/messages`; set `banip.global.ban_debug` to `1` to log every push + ## ts-dns Threat shield DNS (`ts-dns`) is a special configuration for [adblock](https://github.com/openwrt/packages/tree/master/net/adblock). diff --git a/packages/ns-threat_shield/files/banip-insights-defaults b/packages/ns-threat_shield/files/banip-insights-defaults new file mode 100755 index 000000000..7cd946daa --- /dev/null +++ b/packages/ns-threat_shield/files/banip-insights-defaults @@ -0,0 +1,14 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Registered units: run ts-ip after an update to add the Insights hook and feed definition. +# ts-ip preserves a custom block hook, so it runs unless the Insights hook is already set. + +[ -n "$(uci -q get ns-plug.config.secret)" ] || exit 0 +[ "$(uci -q get banip.global.ban_blockhook)" = "/usr/libexec/ts-insights-hook" ] && exit 0 + +/usr/sbin/ts-ip diff --git a/packages/ns-threat_shield/files/banip.nethesis.feeds b/packages/ns-threat_shield/files/banip.nethesis.feeds index 3ec2273a8..4acafe090 100644 --- a/packages/ns-threat_shield/files/banip.nethesis.feeds +++ b/packages/ns-threat_shield/files/banip.nethesis.feeds @@ -23,6 +23,13 @@ "chain": "in", "descr": "Yoroi suspicious - Level 2" }, + "nethesisinsightslvl2": { + "url_4": "https://__USER__:__PASSWORD__@insights.nethesis.it/blocklist/v1/feed", + "url_6": "https://__USER__:__PASSWORD__@insights.nethesis.it/blocklist/v1/feed", + "rule": "feed 1", + "chain": "in", + "descr": "Nethesis community - Level 2" + }, "nethesislvl3": { "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/nethesis_level3.netset", "rule": "feed 1", diff --git a/packages/ns-threat_shield/files/send-insights-blocklist b/packages/ns-threat_shield/files/send-insights-blocklist new file mode 100755 index 000000000..79dbacde2 --- /dev/null +++ b/packages/ns-threat_shield/files/send-insights-blocklist @@ -0,0 +1,144 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Send the IPs blocked by banip, spooled by ts-insights-hook, to Nethesis Insights. +# Run by cron, does nothing if the unit is not registered. +# Only the errors dropping events are logged, unless banip.global.ban_debug is set. + +import base64 +import http.client +import ipaddress +import json +import os +import syslog +import urllib.request + +from euci import EUci + +SPOOL_DIR = "/var/run/ns-insights" +SPOOL = os.path.join(SPOOL_DIR, "threat-events.jsonl") +SENDING = SPOOL + ".sending" + +DEFAULT_URL = "https://insights.nethesis.it/blocklist/v1/events" +MAX_DECISIONS_PER_REQUEST = 500 +MAX_SPOOLED_EVENTS = 5000 +TIMEOUT = 20 + + +def read_config(uci): + """Return (url, system_id, secret), or None if the unit is not registered.""" + system_id = uci.get("ns-plug", "config", "system_id", default="") + secret = uci.get("ns-plug", "config", "secret", default="") + if not system_id or not secret: + return None + + url = uci.get("ns-plug", "config", "insights_blocklist_url", default=DEFAULT_URL) + + return url, system_id, secret + + +def read_spool(): + """Take the spooled lines and remove the spool file.""" + if not os.path.exists(SPOOL): + return [] + + # move the file first, so the hook can keep writing a new one + os.rename(SPOOL, SENDING) + with open(SENDING) as spool: + lines = spool.readlines() + os.unlink(SENDING) + + return lines + + +def is_reportable(value): + """Only public addresses are sent.""" + try: + return ipaddress.ip_address(value).is_global + except ValueError: + return False + + +def parse_events(lines): + """Return the valid and reportable decisions, the most recent ones if over the cap.""" + decisions = [] + + for line in lines: + line = line.strip() + if not line: + continue + try: + decision = json.loads(line) + value = decision["value"] + except (ValueError, KeyError, TypeError): + continue + + if is_reportable(value): + decisions.append(decision) + + if len(decisions) > MAX_SPOOLED_EVENTS: + syslog.syslog( + syslog.LOG_DEBUG, + f"too many events, dropped the {len(decisions) - MAX_SPOOLED_EVENTS} oldest", + ) + decisions = decisions[-MAX_SPOOLED_EVENTS:] + + return decisions + + +def send_batch(config, decisions): + """Post a batch of decisions, any 2xx response means they are queued.""" + url, system_id, secret = config + payload = json.dumps( + {"schema_version": 1, "system_id": system_id, "decisions": decisions} + ).encode() + credentials = base64.b64encode(f"{system_id}:{secret}".encode()).decode() + request = urllib.request.Request( + url, + data=payload, + method="POST", + headers={ + "Content-Type": "application/json", + "Authorization": f"Basic {credentials}", + }, + ) + + with urllib.request.urlopen(request, timeout=TIMEOUT) as response: + response.read() + + +def main(): + syslog.openlog("send-insights-blocklist") + + uci = EUci() + debug = uci.get("banip", "global", "ban_debug", default="0") == "1" + syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG if debug else syslog.LOG_ERR)) + + config = read_config(uci) + if config is None: + return + + decisions = parse_events(read_spool()) + sent = 0 + for index in range(0, len(decisions), MAX_DECISIONS_PER_REQUEST): + batch = decisions[index : index + MAX_DECISIONS_PER_REQUEST] + try: + send_batch(config, batch) + except (http.client.HTTPException, OSError) as error: + syslog.syslog( + syslog.LOG_ERR, + f"push failed, dropped {len(decisions) - index} events: {error}", + ) + break + + sent += len(batch) + + if sent: + syslog.syslog(syslog.LOG_DEBUG, f"pushed {sent} events") + + +if __name__ == "__main__": + main() diff --git a/packages/ns-threat_shield/files/ts-insights-hook b/packages/ns-threat_shield/files/ts-insights-hook new file mode 100755 index 000000000..d76dad593 --- /dev/null +++ b/packages/ns-threat_shield/files/ts-insights-hook @@ -0,0 +1,19 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# banip ban_blockhook: append a blocked IP to the spool, send-insights-blocklist sends it. +# Args: $1 IP; the other banip args ($2 family, $3 log count, $4 Set expiry) are not used + +SPOOL_DIR="/var/run/ns-insights" +SPOOL="${SPOOL_DIR}/threat-events.jsonl" + +[ -n "$1" ] || exit 0 + +[ -d "${SPOOL_DIR}" ] || mkdir -p "${SPOOL_DIR}" + +printf '{"value":"%s","scope":"Ip","type":"ban","scenario":"nethsecurity/banip-log","origin":"banip","created_at":"%s"}\n' \ + "$1" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" >>"${SPOOL}" diff --git a/packages/ns-threat_shield/files/ts-ip b/packages/ns-threat_shield/files/ts-ip index bd4fd7f36..362e6c858 100755 --- a/packages/ns-threat_shield/files/ts-ip +++ b/packages/ns-threat_shield/files/ts-ip @@ -26,6 +26,10 @@ function exit_error { exit 1 } +INSIGHTS_SENDER=/usr/sbin/send-insights-blocklist +INSIGHTS_HOOK=/usr/libexec/ts-insights-hook +INSIGHTS_SPOOL=/var/run/ns-insights/threat-events.jsonl + SYSTEM_SECRET=$(uci -q get ns-plug.config.secret) SYSTEM_ID=$(uci -q get ns-plug.config.system_id) TYPE=$(uci -q get ns-plug.config.type) @@ -46,6 +50,19 @@ if [ ! -z "$SYSTEM_SECRET" ] && [ ! -z "$SYSTEM_ID" ]; then uci add_list banip.global.ban_allowurl="$allow_current" uci commit banip fi + # Nethesis Insights: every subscription reports the blocked IPs, + # the nethesisinsightslvl2 feed is selectable only with the Threat Shield entitlement. + # banip supports a single block hook: a custom one is preserved and disables the reporting + hook=$(uci -q get banip.global.ban_blockhook) + if [ -z "$hook" ]; then + uci set banip.global.ban_blockhook="$INSIGHTS_HOOK" + uci commit banip + elif [ "$hook" != "$INSIGHTS_HOOK" ]; then + logger -t threat_shield "Nethesis Insights reporting disabled: custom banip block hook $hook is configured" + fi + if ! crontab -l 2>/dev/null | grep -q "$INSIGHTS_SENDER"; then + { crontab -l 2>/dev/null; echo "*/5 * * * * sleep \$(( RANDOM % 60 )); $INSIGHTS_SENDER"; } | crontab - + fi else allow=$(uci -q get banip.global.ban_allowurl | tr " " "\n" | grep bl.nethesis.it) if [ "$allow" != "" ]; then @@ -61,4 +78,14 @@ else uci commit banip fi > /etc/banip/banip.custom.feeds + # stop reporting to Nethesis Insights, the feed is removed with the other enterprise feeds; + # a custom block hook is left untouched + if [ "$(uci -q get banip.global.ban_blockhook)" = "$INSIGHTS_HOOK" ]; then + uci -q delete banip.global.ban_blockhook + uci commit banip + fi + if crontab -l 2>/dev/null | grep -q "$INSIGHTS_SENDER"; then + crontab -l | grep -v "$INSIGHTS_SENDER" | crontab - + fi + rm -f "$INSIGHTS_SPOOL" fi diff --git a/packages/ns-threat_shield/files/ts-ip.hook b/packages/ns-threat_shield/files/ts-ip.hook new file mode 100755 index 000000000..c26afde24 --- /dev/null +++ b/packages/ns-threat_shield/files/ts-ip.hook @@ -0,0 +1,13 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# ns-plug register/unregister hook: ts-ip does not reload banip, since it also runs during upgrades + +/usr/sbin/ts-ip +# the banip log service reads the hook, the feeds and the allowlist only on start +/etc/init.d/banip reload +exit 0