banIP: release 1.5.3-1 master
authorDirk Brenken <dev@brenken.org>
Fri, 28 Feb 2025 15:49:39 +0000 (16:49 +0100)
committerDirk Brenken <dev@brenken.org>
Fri, 28 Feb 2025 15:50:56 +0000 (16:50 +0100)
* optimized uci config processing (list options)
* optimized icmp rules in pre-routing (thanks @brada)
* set inbound marker in pre-routing only if inbound logging is enabled (fixes #26044)
* fix cornercase in Set removal function
* print chain-, set- and rules-counter in the banIP status
* clean up logging und download queue handling
* update the readme

Signed-off-by: Dirk Brenken <dev@brenken.org>
net/banip/Makefile
net/banip/files/README.md
net/banip/files/banip-functions.sh
net/banip/files/banip-service.sh

index 6272adae871c44c567d794250e2df577e87dfa2b..65804b16f2806aaf82439e4a4e9ba839cb79d674 100644 (file)
@@ -5,7 +5,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=banip
-PKG_VERSION:=1.5.2
+PKG_VERSION:=1.5.3
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0-or-later
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>
index fe0138a2b50ad4589be229326d2dd33884549853..0b1f947d43e4908a45fc55e103ecaa5789531584 100644 (file)
@@ -163,7 +163,7 @@ Available commands:
 | ban_logreadfile         | option | /var/log/messages             | alternative location for parsing a log file via tail, to deactivate the standard parsing via logread              |
 | ban_autodetect          | option | 1                             | auto-detect wan interfaces, devices and subnets                                                                   |
 | ban_debug               | option | 0                             | enable banIP related debug logging                                                                                |
-| ban_icmplimit           | option | 10                            | threshold in number of packets to detect icmp DoS in prerouting chain. A value of '0' disables this safeguard     |
+| ban_icmplimit           | option | 25                            | threshold in number of packets to detect icmp DoS in prerouting chain. A value of '0' disables this safeguard     |
 | ban_synlimit            | option | 10                            | threshold in number of packets to detect syn DoS in prerouting chain. A value of '0' disables this safeguard      |
 | ban_udplimit            | option | 100                           | threshold in number of packets to detect udp DoS in prerouting chain. A value of '0' disables this safeguard      |
 | ban_logprerouting       | option | 0                             | log supsicious packets in the prerouting chain                                                                    |
@@ -276,19 +276,19 @@ Available commands:
 **banIP runtime information**  
 
 ```
-root@blackhole:~# /etc/init.d/banip status
+~# /etc/init.d/banip status
 ::: banIP runtime information
   + status            : active (nft: ✔, monitor: ✔)
-  + version           : 1.5.0-r3
-  + element_count     : 95820
-  + active_feeds      : cinsscore.v4, country.v6, blocklist.v4, allowlist.v4MAC, allowlist.v6MAC, allowlist.v4, allowlist.v6, country.v4, debl.v4, debl.v6, doh.v4, doh.v6, turris.v4, threat.v4, blocklist.v4MAC, blocklist.v6MAC, blocklist.v6
+  + version           : 1.5.3-r1
+  + element_count     : 96 031 (chains: 7, sets: 18, rules: 46)
+  + active_feeds      : allowlist.v4MAC, allowlist.v6MAC, allowlist.v4, allowlist.v6, cinsscore.v4, country.v6, debl.v4, doh.v6, debl.v6, doh.v4, turris.v6, country.v4, threat.v4, turris.v4, blocklist.v4MAC, blocklist.v6MAC, blocklist.v4, blocklist.v6
   + active_devices    : wan: pppoe-wan / wan-if: wan, wan_6 / vlan-allow: - / vlan-block: -
   + active_uplink     : 91.61.217.158, 2001:fc:37ff:f64:b513:16dd:6903:7710
   + nft_info          : ver: 1.1.1-r1, priority: -100, policy: performance, loglevel: warn, expiry: 2h, limit (icmp/syn/udp): 10/10/100
   + run_info          : base: /mnt/data/banIP, backup: /mnt/data/banIP/backup, report: /mnt/data/banIP/report, error: /mnt/data/banIP/error
   + run_flags         : auto: ✔, proto (4/6): ✔/✔, log (pre/in/out): ✘/✘/✘, count: ✔, dedup: ✔, split: ✘, custom feed: ✘, allowed only: ✘
-  + last_run          : mode: reload, period: 0m 49s, memory: 1388 MB available, 4760 KB max. used, cores: 4, log: logread, fetch: uclient-fetch
-  + system_info       : 2025-01-22 19:10:42, Bananapi BPI-R3, mediatek/filogic, OpenWrt SNAPSHOT r28616-7924acdd63 
+  + last_run          : mode: restart, duration: 0m 19s, memory: 1331.10 MB available, 1.75 MB max. used, cores: 4, log: logread, fetch: curl
+  + system_info       : 2025-02-28 13:29:29, Bananapi BPI-R3, mediatek/filogic, OpenWrt SNAPSHOT r28906-d6977ab33a 
 ```
 
 **banIP search information**  
@@ -478,20 +478,18 @@ Add an unique feed name (no spaces, no special chars) and make the required chan
 Please note: the flag field is optional, it's a space separated list of options: supported are 'gz' as an archive format and protocols 'tcp' or 'udp' with port numbers/port ranges for destination port limitations.  
 
 **Debug options**  
-Whenever you encounter banIP related processing problems, please check the "Processing Log" tab.  
+Whenever you encounter banIP related processing problems, please enable "Verbose Debug Logging", restart banIP and check the "Processing Log" tab.  
 Typical symptoms:  
-* The nftables initialization failed: untick the 'Auto Detection' option in the 'General Settings' config section and set the required options manually  
+* The nftables initialization failed: untick the 'Auto Detection' option in the 'General Settings' config section and set the required device and tools options manually  
 * A blocklist feed does not work: maybe a temporary server problem or the download URL has been changed. In the latter case, just use the Custom Feed Editor to point this feed to a new URL  
 
-To get much more processing information, please enable "Verbose Debug Logging" and restart banIP.  
-
 In case of a nft processing error, banIP creates an error directory (by default '/tmp/banIP-error') with the faulty nft load files.  
 For further troubleshooting, you can try to load such an error file manually to determine the exact cause of the error, e.g.: 'nft -f error.file.nft'.  
 
 Whenever you encounter firewall problems, enable the logging of certain chains in the "Log Settings" config section, restart banIP and check the "Firewall Log" tab.  
 Typical symptoms:  
 * A feed blocks a legit IP: disable the entire feed or add this IP to your local allowlist and reload banIP  
-* A feed (e.g. doh) interrupts almost all client connections: check the feed table above for reference and limit the feed to a certain chain in the "Feed/Set Settings" config section  
+* A feed (e.g. doh) interrupts almost all client connections: check the feed table above for reference and reset the feed to the defaults in the "Feed/Set Settings" config tab section  
 * The allowlist doesn't free a certain IP/MAC address: check the current content of the allowlist with the "Set Survey" under the "Set Reporting" tab to make sure that the desired IP/MAC is listed - if not, reload banIP  
 
 <a id="support"></a>
index ec99a07bdd01e3c0280d48db22d571b4fb9da3f6..940765394beb4a88ed2518cdb832c30c309d8d53 100644 (file)
@@ -42,7 +42,7 @@ ban_nftpolicy="memory"
 ban_nftexpiry=""
 ban_nftretry="5"
 ban_nftcount="0"
-ban_icmplimit="10"
+ban_icmplimit="25"
 ban_synlimit="10"
 ban_udplimit="100"
 ban_loglimit="100"
@@ -257,7 +257,6 @@ f_log() {
 f_conf() {
        local rir ccode region country
 
-       unset ban_dev ban_vlanallow ban_vlanblock ban_ifv4 ban_ifv6 ban_feed ban_allowurl ban_feedin ban_feedout ban_feedinout ban_feedreset ban_feedcomplete ban_logterm ban_region ban_country ban_asn
        config_cb() {
                option_cb() {
                        local option="${1}" value="${2//\"/\\\"}"
@@ -265,64 +264,14 @@ f_conf() {
                        eval "${option}=\"${value}\""
                }
                list_cb() {
-                       local option="${1}" value="${2//\"/\\\"}"
+                       local append option="${1}" value="${2//\"/\\\"}"
 
-                       case "${option}" in
-                               "ban_ifv4")
-                                       eval "${option}=\"$(printf "%s" "${ban_ifv4}")${value} \""
-                                       ;;
-                               "ban_ifv6")
-                                       eval "${option}=\"$(printf "%s" "${ban_ifv6}")${value} \""
-                                       ;;
-                               "ban_dev")
-                                       eval "${option}=\"$(printf "%s" "${ban_dev}")${value} \""
-                                       ;;
-                               "ban_vlanallow")
-                                       eval "${option}=\"$(printf "%s" "${ban_vlanallow}")${value} \""
-                                       ;;
-                               "ban_vlanblock")
-                                       eval "${option}=\"$(printf "%s" "${ban_vlanblock}")${value} \""
-                                       ;;
-                               "ban_trigger")
-                                       eval "${option}=\"$(printf "%s" "${ban_trigger}")${value} \""
-                                       ;;
-                               "ban_feed")
-                                       eval "${option}=\"$(printf "%s" "${ban_feed}")${value} \""
-                                       ;;
-                               "ban_feedin")
-                                       eval "${option}=\"$(printf "%s" "${ban_feedin}")${value} \""
-                                       ;;
-                               "ban_feedout")
-                                       eval "${option}=\"$(printf "%s" "${ban_feedout}")${value} \""
-                                       ;;
-                               "ban_feedinout")
-                                       eval "${option}=\"$(printf "%s" "${ban_feedinout}")${value} \""
-                                       ;;
-                               "ban_feedreset")
-                                       eval "${option}=\"$(printf "%s" "${ban_feedreset}")${value} \""
-                                       ;;
-                               "ban_feedcomplete")
-                                       eval "${option}=\"$(printf "%s" "${ban_feedcomplete}")${value} \""
-                                       ;;
-                               "ban_allowurl")
-                                       eval "${option}=\"$(printf "%s" "${ban_allowurl}")${value} \""
-                                       ;;
-                               "ban_logterm")
-                                       eval "${option}=\"$(printf "%s" "${ban_logterm}")${value}\\|\""
-                                       ;;
-                               "ban_region")
-                                       eval "${option}=\"$(printf "%s" "${ban_region}")${value} \""
-                                       ;;
-                               "ban_country")
-                                       eval "${option}=\"$(printf "%s" "${ban_country}")${value} \""
-                                       ;;
-                               "ban_asn")
-                                       eval "${option}=\"$(printf "%s" "${ban_asn}")${value} \""
-                                       ;;
-                       esac
+                       eval "append=\"\${${option}}\""
+                       eval "${option}=\"${append}${value} \""
                }
        }
        config_load banip
+
        [ -f "${ban_logreadfile}" ] && ban_logreadcmd="$(command -v tail)" || ban_logreadcmd="$(command -v logread)"
 
        for rir in ${ban_region}; do
@@ -702,10 +651,7 @@ f_nftinit() {
                #
                printf "%s\n" "add rule inet banIP pre-routing iifname != { ${wan_dev} } counter accept"
                printf "%s\n" "add rule inet banIP pre-routing ct state invalid ${log_ct} counter name cnt_ctinvalid drop"
-               if [ "${ban_icmplimit}" -gt "0" ]; then
-                       printf "%s\n" "add rule inet banIP pre-routing ip protocol icmp limit rate over ${ban_icmplimit}/second ${log_icmp} counter name cnt_icmpflood drop"
-                       printf "%s\n" "add rule inet banIP pre-routing ip6 nexthdr icmpv6 limit rate over ${ban_icmplimit}/second ${log_icmp} counter name cnt_icmpflood drop"
-               fi
+               [ "${ban_icmplimit}" -gt "0" ] && printf "%s\n" "add rule inet banIP pre-routing meta nfproto . meta l4proto { ipv4 . icmp , ipv6 . icmpv6 } limit rate over ${ban_icmplimit}/second ${log_icmp} counter name cnt_icmpflood drop"
                [ "${ban_udplimit}" -gt "0" ] && printf "%s\n" "add rule inet banIP pre-routing meta l4proto udp ct state new limit rate over ${ban_udplimit}/second ${log_udp} counter name cnt_udpflood drop"
                [ "${ban_synlimit}" -gt "0" ] && printf "%s\n" "add rule inet banIP pre-routing tcp flags & (fin|syn|rst|ack) == syn limit rate over ${ban_synlimit}/second ${log_syn} counter name cnt_synflood drop"
                printf "%s\n" "add rule inet banIP pre-routing tcp flags & (fin|syn) == (fin|syn) ${log_tcp} counter name cnt_tcpinvalid drop"
@@ -722,16 +668,14 @@ f_nftinit() {
                printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 icmpv6 type { nd-neighbor-advert, nd-neighbor-solicit, nd-router-advert} ip6 hoplimit 1 counter accept"
                printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 icmpv6 type { nd-neighbor-advert, nd-neighbor-solicit, nd-router-advert} ip6 hoplimit 255 counter accept"
                [ -n "${allow_dport}" ] && printf "%s\n" "add rule inet banIP wan-input ${allow_dport} counter accept"
-               printf "%s\n" "add rule inet banIP wan-input meta mark set 1"
-               printf "%s\n" "add rule inet banIP wan-input counter jump _inbound"
+               [ "${ban_loginbound}" = "1" ] && printf "%s\n" "add rule inet banIP wan-input meta mark set 1 counter jump _inbound" || printf "%s\n" "add rule inet banIP wan-input counter jump _inbound"
 
                # default wan-forward rules
                #
                printf "%s\n" "add rule inet banIP wan-forward iifname != { ${wan_dev} } counter accept"
                printf "%s\n" "add rule inet banIP wan-forward ct state established,related counter accept"
                [ -n "${allow_dport}" ] && printf "%s\n" "add rule inet banIP wan-forward ${allow_dport} counter accept"
-               printf "%s\n" "add rule inet banIP wan-forward meta mark set 2"
-               printf "%s\n" "add rule inet banIP wan-forward counter jump _inbound"
+               [ "${ban_loginbound}" = "1" ] && printf "%s\n" "add rule inet banIP wan-forward meta mark set 2 counter jump _inbound" || printf "%s\n" "add rule inet banIP wan-forward counter jump _inbound"
 
                # default lan-forward rules
                #
@@ -1275,14 +1219,16 @@ f_rmset() {
                                        "country")
                                                country="${feed%.*}"
                                                country="${country#*.}"
-                                               if [ "${ban_countrysplit}" = "1" ] && printf "%s" "${ban_country}" | "${ban_grepcmd}" -q "${country}"; then
+                                               if [ "${ban_countrysplit}" = "1" ] && printf "%s" "${ban_feed}" | "${ban_grepcmd}" -q "${feed%%.*}" &&
+                                                       printf "%s" "${ban_country}" | "${ban_grepcmd}" -q "${country}"; then
                                                        continue
                                                fi
                                                ;;
                                        asn)
                                                asn="${feed%.*}"
                                                asn="${asn#*.}"
-                                               if [ "${ban_asnsplit}" = "1" ] && printf "%s" "${ban_asn}" | "${ban_grepcmd}" -q "${asn}"; then
+                                               if [ "${ban_asnsplit}" = "1" ] && printf "%s" "${ban_feed}" | "${ban_grepcmd}" -q "${feed%%.*}" &&
+                                                       printf "%s" "${ban_asn}" | "${ban_grepcmd}" -q "${asn}"; then
                                                        continue
                                                fi
                                                ;;
@@ -1316,7 +1262,7 @@ f_rmset() {
 # generate status information
 #
 f_genstatus() {
-       local mem_free mem_max nft_ver object end_time duration table_sets cnt_elements="0" custom_feed="0" split="0" status="${1}"
+       local mem_free mem_max nft_ver chain_cnt set_cnt rule_cnt object end_time duration table table_sets element_cnt="0" custom_feed="0" split="0" status="${1}"
 
        mem_free="$("${ban_awkcmd}" '/^MemAvailable/{printf "%.2f", $2/1024}' "/proc/meminfo" 2>/dev/null)"
        mem_max="$("${ban_awkcmd}" '/^VmHWM/{printf "%.2f", $2/1024}' /proc/${$}/status 2>/dev/null)"
@@ -1324,16 +1270,20 @@ f_genstatus() {
 
        [ -z "${ban_dev}" ] && f_conf
        if [ "${status}" = "active" ]; then
+               table="$("${ban_nftcmd}" -tj list table inet banIP 2>/dev/null)"
+               table_sets="$(printf "%s" "${table}" | "${ban_jsoncmd}" -qe '@.nftables[@.set.family="inet"].set.name')"
+               for object in ${table_sets}; do
+                       element_cnt="$((element_cnt + $("${ban_nftcmd}" -j list set inet banIP "${object}" 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]' | "${ban_wccmd}" -l 2>/dev/null)))"
+               done
+               chain_cnt="$(printf "%s" "${table}" | "${ban_jsoncmd}" -qe '@.nftables[*].chain.name' | "${ban_wccmd}" -l 2>/dev/null)"
+               set_cnt="$(printf "%s" "${table}" | "${ban_jsoncmd}" -qe '@.nftables[*].set.name' | "${ban_wccmd}" -l 2>/dev/null)"
+               rule_cnt="$(printf "%s" "${table}" | "${ban_jsoncmd}" -qe '@.nftables[*].rule' | "${ban_wccmd}" -l 2>/dev/null)"
+               element_cnt="$("${ban_awkcmd}" -v cnt="${element_cnt}" 'BEGIN{res="";pos=0;for(i=length(cnt);i>0;i--){res=substr(cnt,i,1)res;pos++;if(pos==3&&i>1){res=" "res;pos=0;}}; printf"%s",res}')"
                if [ -n "${ban_starttime}" ] && [ "${ban_action}" != "boot" ]; then
                        end_time="$(date "+%s")"
                        duration="$(((end_time - ban_starttime) / 60))m $(((end_time - ban_starttime) % 60))s"
                fi
-               table_sets="$("${ban_nftcmd}" -tj list table inet banIP 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[@.set.family="inet"].set.name')"
-               for object in ${table_sets}; do
-                       cnt_elements="$((cnt_elements + $("${ban_nftcmd}" -j list set inet banIP "${object}" 2>/dev/null | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]' | "${ban_wccmd}" -l 2>/dev/null)))"
-               done
-               cnt_elements="$("${ban_awkcmd}" -v cnt="${cnt_elements}" 'BEGIN{res="";pos=0;for(i=length(cnt);i>0;i--){res=substr(cnt,i,1)res;pos++;if(pos==3&&i>1){res=" "res;pos=0;}}; printf"%s",res}')"
-               runtime="mode: ${ban_action:-"-"}, period: ${duration:-"-"}, memory: ${mem_free} MB available, ${mem_max} MB max. used, cores: ${ban_cores}, log: ${ban_logreadcmd##*/}, fetch: ${ban_fetchcmd##*/}"
+               runtime="mode: ${ban_action:-"-"}, duration: ${duration:-"-"}, memory: ${mem_free} MB available, ${mem_max} MB max. used, cores: ${ban_cores}, log: ${ban_logreadcmd##*/}, fetch: ${ban_fetchcmd##*/}"
        fi
        [ -s "${ban_customfeedfile}" ] && custom_feed="1"
        [ "${ban_splitsize:-"0"}" -gt "0" ] && split="1"
@@ -1343,7 +1293,7 @@ f_genstatus() {
        json_load_file "${ban_rtfile}" >/dev/null 2>&1
        json_add_string "status" "${status}"
        json_add_string "version" "${ban_ver}"
-       json_add_string "element_count" "${cnt_elements}"
+       json_add_string "element_count" "${element_cnt} (chains: ${chain_cnt:-"0"}, sets: ${set_cnt:-"0"}, rules: ${rule_cnt:-"0"})"
        json_add_array "active_feeds"
        for object in ${table_sets:-"-"}; do
                json_add_string "${object}" "${object}"
@@ -1462,7 +1412,7 @@ f_lookup() {
        end_time="$(date "+%s")"
        duration="$(((end_time - start_time) / 60))m $(((end_time - start_time) % 60))s"
 
-       f_log "info" "domain lookup finished in ${duration} (${feed}, ${cnt_domain} domains, ${cnt_ip} IPs)"
+       f_log "debug" "f_lookup    ::: feed: ${feed}, domains: ${cnt_domain}, IPs: ${cnt_ip}, duration: ${duration}"
 }
 
 # table statistics
@@ -1707,7 +1657,7 @@ f_search() {
                        fi
                ) &
                hold="$((cnt % ban_cores))"
-               [ "${hold}" = "0" ] && wait
+               [ "${hold}" = "0" ] && wait -n
                cnt="$((cnt + 1))"
        done
        wait
@@ -1767,7 +1717,6 @@ f_mail() {
        #
        ban_mailhead="From: ${ban_mailsender}\nTo: ${ban_mailreceiver}\nSubject: ${ban_mailtopic}\nReply-to: ${ban_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n"
        printf "%b" "${ban_mailhead}${mail_text}" | "${ban_mailcmd}" --timeout=10 ${msmtp_debug} -a "${ban_mailprofile}" "${ban_mailreceiver}" >/dev/null 2>&1
-       f_log "info" "send status mail (${?})"
 
        f_log "debug" "f_mail      ::: notification: ${ban_mailnotification}, template: ${ban_mailtemplate}, profile: ${ban_mailprofile}, receiver: ${ban_mailreceiver}, rc: ${?}"
 }
index 2850c772137a6166178178ac5d3a232d0e47af77..4dbaed78cb24651d6b85ac0fb5d5f12393a4fe05 100755 (executable)
@@ -98,10 +98,10 @@ for feed in allowlist ${ban_feed} blocklist; do
                fi
                if [ "${feed_url_4}" = "${feed_url_6}" ]; then
                        feed_url_6="local"
-                       wait
+                       wait -n
                else
                        hold="$((cnt % ban_cores))"
-                       [ "${hold}" = "0" ] && wait
+                       [ "${hold}" = "0" ] && wait -n
                        cnt="$((cnt + 1))"
                fi
        fi
@@ -119,10 +119,9 @@ for feed in allowlist ${ban_feed} blocklist; do
                fi
                cnt="$((cnt + 1))"
                hold="$((cnt % ban_cores))"
-               [ "${hold}" = "0" ] && wait
+               [ "${hold}" = "0" ] && wait -n
        fi
 done
-wait
 f_rmset
 f_rmdir "${ban_tmpdir}"
 f_genstatus "active"
@@ -134,13 +133,14 @@ cnt="1"
 for list in allowlist blocklist; do
        (f_lookup "${list}") &
        hold="$((cnt % ban_cores))"
-       [ "${hold}" = "0" ] && wait
+       [ "${hold}" = "0" ] && wait -n
        cnt="$((cnt + 1))"
 done
 wait
 
 # end processing
 #
+f_log "info" "finish banIP processing"
 (
        sleep 5
        if [ "${ban_mailnotification}" = "1" ] && [ -n "${ban_mailreceiver}" ] && [ -x "${ban_mailcmd}" ]; then