From bb5f4f6d2972df2b67c6b3ee27faa190e3be0a28 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Sat, 30 Apr 2016 22:18:25 +0200 Subject: [PATCH] adblock: release 1.1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit * support more router modes, as long as the firewall and the DNS server are enabled * new source sites can be added & changed via awk ruleset in uci config * source domain count, last update time & overall count will be stored in uci config * added 3 new source sites: ransomware tracker rolist/easylist winspy * switch to minimal inline base64 encoded 1×1 GIF for pixel server (separate png image no longer needed) * simplified uci parser * source download & domain sort optimization * add whitelist parser with wildcard support * reduced code size & various cleanups * updated documentation Signed-off-by: Dirk Brenken (cherry picked from commit b9aeb82c957425c2cb84738833813684731df503) --- net/adblock/Makefile | 4 +- net/adblock/files/README.md | 126 +++++-- net/adblock/files/adblock-helper.sh | 393 ++++++++++------------ net/adblock/files/adblock-update.sh | 231 ++++--------- net/adblock/files/adblock.conf | 118 ++++--- net/adblock/files/adblock.init | 4 + net/adblock/files/www/adblock/adblock.png | Bin 68 -> 0 bytes net/adblock/files/www/adblock/index.html | 5 +- 8 files changed, 442 insertions(+), 439 deletions(-) delete mode 100644 net/adblock/files/www/adblock/adblock.png diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 2333a11343..5597daea69 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=1.0.3 +PKG_VERSION:=1.1.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken @@ -24,7 +24,7 @@ endef define Package/$(PKG_NAME)/description Powerful adblock script to block ad/abuse domains. -Currently the script supports 16 domain blacklist sites plus manual black- and whitelist overrides. +Currently the script supports 19 domain blacklist sites plus manual black- and whitelist overrides. Please see https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md for further information. endef diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 3c90e1ca5e..7e0c171b5f 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -7,6 +7,8 @@ A lot of people already use adblocker plugins within their desktop browsers, but * support of the following domain blocklist sources (free for private usage, for commercial use please check their individual licenses): * [adaway](https://adaway.org) * => infrequent updates, approx. 400 entries (enabled by default) + * [blacklist]() + * => static local blacklist, located by default in '/etc/adblock/adblock.blacklist' * [disconnect](https://disconnect.me) * => numerous updates on the same day, approx. 6.500 entries (enabled by default) * [dshield](http://dshield.org) @@ -19,8 +21,12 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => daily updates, approx. 1.500 entries * [openphish](https://openphish.com) * => numerous updates on the same day, approx. 1.800 entries - * [palevotracker](https://palevotracker.abuse.ch) + * [palevo tracker](https://palevotracker.abuse.ch) * => daily updates, approx. 15 entries + * [ransomware tracker](https://ransomwaretracker.abuse.ch) + * => daily updates, approx. 150 entries + * [rolist/easylist](https://easylist-downloads.adblockplus.org/rolist+easylist.txt) + * => weekly updates, approx. 600 entries * [ruadlist/easylist](https://code.google.com/p/ruadlist) * => weekly updates, approx. 2.000 entries * [shallalist](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default) @@ -33,82 +39,144 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => weekly updates, approx. 12.000 entries * [winhelp](http://winhelp2002.mvps.org) * => infrequent updates, approx. 15.000 entries + * [winspy](https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/hostsBlockWindowsSpy.txt) + * => infrequent updates, approx. 120 entries * [yoyo](http://pgl.yoyo.org/adservers) * => weekly updates, approx. 2.500 entries (enabled by default) - * [zeustracker](https://zeustracker.abuse.ch) + * [zeus tracker](https://zeustracker.abuse.ch) * => daily updates, approx. 440 entries * zero-conf like automatic installation & setup, usually no manual changes needed (i.e. ip address, network devices etc.) +* supports a wide range of router modes, as long as the firewall and the DNS server are enabled * full IPv4 and IPv6 support * each blocklist source will be updated and processed separately * timestamp check to download and process only updated adblock list sources * overall duplicate removal in separate adblock lists (will be automatically disabled on low memory systems) * adblock source list parsing by fast & flexible regex rulesets -* additional white- and blacklist support for manual overrides +* additional whitelist for manual overrides, located by default in /etc/adblock/adblock.whitelist * quality checks during & after update of adblock lists to ensure a reliable dnsmasq service -* basic adblock statistics via iptables packet counters for each chain +* basic adblock statistics via iptables packet counters +* list states, (overall) list counts & last update time will be stored in uci config * status & error logging to stdout and syslog * use a dynamic uhttpd instance as an adblock pixel server * use dynamic iptables rulesets for adblock related redirects/rejects * openwrt init system support (start/stop/restart/reload) * hotplug support, the adblock start will be triggered by wan 'ifup' event * optional: adblock list backup/restore (disabled by default) +* optional: add new adblock sources via uci config (see example below) ## Prerequisites -* [openwrt](https://openwrt.org), tested with latest stable release (Chaos Calmer 15.05) and with current trunk (Designated Driver > r47025) -* usual openwrt setup with 'iptables' & 'uhttpd', additional required software packages: +* [openwrt](https://openwrt.org), tested with latest stable release (Chaos Calmer) and with current trunk (Designated Driver) +* usual openwrt setup with enabled 'iptables', 'dnsmasq' and 'uhttpd' - dump AP modes are _not_ supported! +* additional required software packages: * wget * optional: 'kmod-ipt-nat6' for IPv6 support * the above dependencies and requirements will be checked during package installation & script runtime ## Designated Driver Installation & Usage -* install the adblock package (*opkg install adblock*) -* start the adblock service with */etc/init.d/adblock start* and check *logread -e "adblock"* for adblock related information -* optional: enable/disable your required adblock list sources in */etc/config/adblock* - 'adaway', 'disconnect' and 'yoyo' are enabled by default +* install 'adblock' (_opkg install adblock_) +* adblock starts automatically during boot/wan-ifup event, check _logread -e "adblock"_ for adblock related information +* optional: start/restart/stop the adblock service manually with _/etc/init.d/adblock_ +* optional: enable/disable your required adblock list sources in _/etc/config/adblock_ - 'adaway', 'disconnect' and 'yoyo' are enabled by default * optional: maintain the adblock service in luci under 'System => Startup' ## LuCI adblock companion package -For easy management of the various blocklist sources and and the adblock options there is also a nice & efficient LuCI frontend available. -Please install the package 'luci-app-adblock' (*opkg install luci-app-adblock*). Then you will find the application in LuCI located under 'Services' menu. -Thanks to Hannu Nyman for this great adblock LuCI frontend! +* for easy management of the various blocklist sources and adblock options there is also a nice & efficient LuCI frontend available +* install 'luci-app-adblock' (_opkg install luci-app-adblock_) +* the application is located in LuCI under 'Services' menu +* _Thanks to Hannu Nyman for this great adblock LuCI frontend!_ ## Chaos Calmer installation notes -* currently the adblock package is *not* part of the CC package repository -* download the latest adblock package *adblock_x.xx.x-1_all.ipk* from a development snapshot [package directory](https://downloads.openwrt.org/snapshots/trunk/ar71xx/nand/packages/packages) -* due to server hardware troubles the package directory link above may not work, if so please check the [main openwrt download area](https://downloads.openwrt.org) manually -* manual transfer the package to your router and install the opkg package as usual +* 'adblock' and 'luci-app-adblock' are _not_ available as .ipk packages in the Chaos Calmer download repository +* download both packages from a development snapshot package directory: + * for 'adblock' look [here](https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/packages/packages/) + * for 'luci-app-adblock' look [here](https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/packages/luci/) +* manually transfer the packages to your routers temp directory (with tools like _sshfs_ or _winscp_) +* install the packages with _opkg install <...>_ as described above ## Tweaks -* there is no need to enable all blacklist sites at once, for normal use one to three adblock list sources should be sufficient -* if you really need to handle all blacklists at once add an usb stick or any other storage device to enlarge your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details -* add personal domain white- or blacklist entries as an additional blocklist source, one domain per line (wildcards & regex are not allowed!), by default both empty lists are located in */etc/adblock* -* enable the backup/restore feature, to restore automatically the latest stable backup of your adblock lists in case of any (partial) processing error (i.e. a single blocklist source server is down). Please use an (external) solid partition and *not* your volatile router temp directory for this -* for a scheduled call of the adblock service via */etc/init.d/adblock start* add an appropriate crontab entry -* in case of any script runtime errors, you should enable script debugging: for this please change the value of the main 'DEBUG' switch, you'll find it in the header of */usr/bin/adblock-update.sh* +* to process & store all blocklist sources at once it might helpful to enlarge your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details +* add domain white- or blacklist entries to always-allow or -deny certain (sub) domains, by default both lists are located in _/etc/adblock_. Please add one domain per line - ip addresses, wildcards & regex are _not_ allowed (see example below) +* enable the backup/restore feature, to restore automatically the latest stable backup of your adblock lists in case of any (partial) processing error (i.e. a single blocklist source is down). Please use an (external) solid partition and _not_ your volatile router temp directory for this +* for a scheduled call of the adblock service via _/etc/init.d/adblock start_ add an appropriate crontab entry (see example below) +* in case of any script runtime errors, you should enable script debugging: for this please change the 'DEBUG' variable in the header of _/usr/bin/adblock-update.sh_ from '0' to '1' and start this script afterwards (without any parameter) +* you could add new blocklist sources on your own via uci config, all you need is a source url and an awk one-liner (see example below) ## Further adblock config options -* usually the adblock autodetection works quite well and no manual config overrides are needed, all options apply to 'global' adblock config section: +* usually the adblock autodetection works quite well and no manual config overrides are needed, all options apply to the 'global' config section: * adb\_enabled => main switch to enable/disable adblock service (default: '1', enabled) - * adb\_cfgver => config version string (do not change!) - adblock checks this entry and automatically applies the current config, if none or an older revision was found. - * adb\_wanif => name of the logical wan interface (default: 'wan') + * adb\_cfgversion => config version string (do not change!) - adblock will check this entry during startup * adb\_lanif => name of the logical lan interface (default: 'lan') - * adb\_port => port of the adblock uhttpd instance (default: '65535') + * adb\_nullport => port of the adblock uhttpd instance (default: '65535') * adb\_nullipv4 => IPv4 blackhole ip address (default: '192.0.2.1') * adb\_nullipv6 => IPv6 blackhole ip address (default: '::ffff:c000:0201') * adb\_forcedns => redirect all DNS queries to local dnsmasq resolver (default: '1', enabled) +## Examples + +**example cronjob for a regular update of the block lists:** +

+# configuration found in /etc/crontabs/root
+# start adblock script once a day at 6 a.m.
+#
+0 06 * * *    /etc/init.d/adblock start
+
+ +**example blacklist entry (/etc/adblock/adblock.blacklist):** +

+ads.example.com
+
+ +This rule blocks: +http://ads.example.com/foo.gif +http://server1.ads.example.com/foo.gif +https://ads.example.com:8000/ + +This rule doesn't block: +http://ads.example.com.ua/foo.gif +http://example.com/ + +**example whitelist entry (/etc/adblock/adblock.whitelist):** +

+analytics.com
+
+ +This rule removes _all_ domains from the blocklists with this string in it, i.e.: + google-analytics.com + ssl.google-analytics.com + api.gameanalytics.com + photos.daily-deals.analoganalytics.com + adblockanalytics.com + +**example to add [rolist+easylist](https://easylist-downloads.adblockplus.org/rolist+easylist.txt) as a new source:** +

+add a new source section in adblock config, take an existing easylist source as a template and change only the source name, the url and the description
+
+config source 'ruadlist'
+        option enabled '0'
+        option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt'
+        option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}'
+        option adb_src_desc 'focus on russian ad related domains plus generic easylist additions, weekly updates, approx. 2.000 entries'
+
+config source 'rolist'
+        option enabled '0'
+        option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt'
+        option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}'
+        option adb_src_desc 'focus on romanian ad related domains plus generic easylist additions, weekly updates, approx. 600 entries'
+
+ ## Background This adblock package is a dns/dnsmasq based adblock solution for openwrt. Queries to ad/abuse domains are never forwarded and always replied with a local IP address which may be IPv4 or IPv6. For that purpose adblock uses an ip address from the private 'TEST-NET-1' subnet (192.0.2.1 / ::ffff:c000:0201) by default. Furthermore all ad/abuse queries will be filtered by ip(6)tables and redirected to internal adblock pixel server (in PREROUTING chain) or rejected (in FORWARD or OUTPUT chain). -All iptables and uhttpd related adblock additions are non-destructive, no hard-coded changes in 'firewall.user', 'uhttpd' config or any other openwrt related config files. There is *no* adblock background daemon running, the (scheduled) start of the adblock service keeps only the adblock lists up-to-date. +All iptables and uhttpd related adblock additions are non-destructive, no hard-coded changes in 'firewall.user', 'uhttpd' config or any other openwrt related config files. There is _no_ adblock background daemon running, the (scheduled) start of the adblock service keeps only the adblock lists up-to-date. ## Support Please join the adblock discussion in this [openwrt forum thread](https://forum.openwrt.org/viewtopic.php?id=59803) or contact me by mail ## Removal -* stop all adblock related services with */etc/init.d/adblock stop* -* optional: remove the adblock package (*opkg remove adblock*) +* stop all adblock related services with _/etc/init.d/adblock stop_ +* optional: remove the adblock package (_opkg remove adblock_) Have fun! Dirk diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh index 246f98b960..c99f58d56e 100644 --- a/net/adblock/files/adblock-helper.sh +++ b/net/adblock/files/adblock-helper.sh @@ -2,36 +2,10 @@ # function library used by adblock-update.sh # written by Dirk Brenken (openwrt@brenken.org) -##################################### # f_envload: load adblock environment # f_envload() { - local cfg_version - - # get version string from default adblock configuration file - # - cfg_version="$(/sbin/uci -q get adblock.global.adb_cfgver)" - cfg_enabled="$(/sbin/uci -q get adblock.global.adb_enabled)" - rc=$? - if [ $((rc)) -ne 0 ] || [ "${cfg_version}" != "${adb_scriptver%.*}" ] - then - cp -pf "/etc/adblock/adblock.conf.default" "/etc/config/adblock" - rc=$? - if [ $((rc)) -eq 0 ] - then - f_log "new default adblock configuration applied, please check your settings in '/etc/config/adblock'" - else - f_log "original adblock configuration not found, please (re-)install the adblock package via 'opkg install adblock --force-maintainer'" "${rc}" - f_exit - fi - elif [ $((rc)) -eq 0 ] && [ $((cfg_enabled)) -ne 1 ] - then - rc=-1 - f_log "adblock is currently disabled, please run 'uci set adblock.global.adb_enabled=1' and 'uci commit adblock' to enable this service" - f_exit - fi - # source in openwrt function library # if [ -r "/lib/functions.sh" ] @@ -54,46 +28,22 @@ f_envload() f_exit fi - # check opkg availability and get list with all installed openwrt packages - # - if [ -r "/var/lock/opkg.lock" ] - then - rc=-1 - f_log "adblock installation finished, 'opkg' currently locked by package installer" - f_exit - fi - pkg_list="$(opkg list-installed)" - if [ -z "${pkg_list}" ] - then - rc=120 - f_log "empty openwrt package list" "${rc}" - f_exit - fi -} - -###################################################### -# f_envparse: parse adblock config and set environment -# -f_envparse() -{ # set initial defaults, # may be overwritten by setting appropriate adblock config options in global section of /etc/config/adblock # - adb_wanif="wan" adb_lanif="lan" - adb_port="65535" + adb_nullport="65535" adb_nullipv4="192.0.2.1" adb_nullipv6="::ffff:c000:0201" - adb_blacklist="/etc/adblock/adblock.blacklist" adb_whitelist="/etc/adblock/adblock.whitelist" + adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}" adb_forcedns=1 - # function to read global options by callback + # function to parse global section by callback # config_cb() { local type="${1}" - local name="${2}" if [ "${type}" = "adblock" ] then option_cb() @@ -107,125 +57,107 @@ f_envparse() fi } - # function to iterate through config list, read only options in "enabled" sections + # function to parse 'service' and 'source' sections # - adb_cfglist="adb_backupdir adb_src" - unset adb_sources parse_config() { - local config="${1}" - config_get switch "${config}" "enabled" + local value opt section="${1}" options="adb_dir adb_src adb_src_rset adb_src_cat" + config_get switch "${section}" "enabled" if [ "${switch}" = "1" ] then - for option in ${adb_cfglist} + if [ "${section}" != "backup" ] + then + eval "adb_sources=\"${adb_sources} ${section}\"" + fi + for opt in ${options} do - config_get value "${config}" "${option}" + config_get value "${section}" "${opt}" if [ -n "${value}" ] then - if [ "${option}" = "adb_src" ] - then - if [ "${config}" = "shalla" ] - then - categories() - { - local cat="${1}" - adb_cat_shalla="${adb_cat_shalla} ${cat}" - } - eval "adb_arc_shalla=\"${value}\"" - config_list_foreach "shalla" "adb_catlist" "categories" - else - adb_sources="${adb_sources} ${value}" - fi - else - eval "${option}=\"${value}\"" - fi + eval "${opt}_${section}=\"${value}\"" fi done fi } + # check opkg availability + # + if [ -r "/var/lock/opkg.lock" ] + then + rc=-1 + f_log "adblock installation finished successfully, 'opkg' currently locked by package installer" + f_exit + fi + + # get list with all installed openwrt packages + # + pkg_list="$(opkg list-installed)" + if [ -z "${pkg_list}" ] + then + rc=120 + f_log "empty openwrt package list" "${rc}" + f_exit + fi + # load adblock config and start parsing functions # config_load adblock config_foreach parse_config service config_foreach parse_config source + # check 'enabled' & 'version' config options + # + if [ -z "${adb_enabled}" ] || [ -z "${adb_cfgversion}" ] || [ "${adb_cfgversion}" != "${adb_scriptver%.*}" ] + then + rc=125 + f_log "no valid adblock configuration found, please merge latest changes from '/etc/config/adblock.opkg' manually" "${rc}" + f_exit + fi + if [ $((adb_enabled)) -ne 1 ] + then + rc=-1 + f_log "adblock is currently disabled, please set adblock.global.adb_enabled=1' to use this service" + f_exit + fi + # set more script defaults (can't be overwritten by adblock config options) # - adb_count=0 adb_minspace=12000 adb_tmpfile="$(mktemp -tu)" adb_tmpdir="$(mktemp -p /tmp -d)" adb_dnsdir="/tmp/dnsmasq.d" adb_dnsprefix="adb_list" - adb_prechain_ipv4="prerouting_rule" - adb_fwdchain_ipv4="forwarding_rule" - adb_outchain_ipv4="output_rule" - adb_prechain_ipv6="PREROUTING" - adb_fwdchain_ipv6="forwarding_rule" - adb_outchain_ipv6="output_rule" adb_fetch="/usr/bin/wget" unset adb_srclist adb_revsrclist adb_errsrclist - # set adblock source ruleset definitions - # - rset_core="([A-Za-z0-9_-]+\.){1,}[A-Za-z]+" - rset_adaway="awk '\$0 ~/^127\.0\.0\.1[ \t]+${rset_core}/{print tolower(\$2)}'" - rset_blacklist="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_disconnect="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_dshield="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_feodo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_malware="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_malwarelist="awk '\$0 ~/^127\.0\.0\.1[ \t]+${rset_core}/{print tolower(\$2)}'" - rset_openphish="awk -F '/' '\$3 ~/^${rset_core}/{print tolower(\$3)}'" - rset_palevo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_ruadlist="awk -F '[|^]' '\$0 ~/^\|\|${rset_core}\^$/{print tolower(\$3)}'" - rset_shalla="awk -F '/' '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_spam404="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_sysctl="awk '\$0 ~/^127\.0\.0\.1[ \t]+${rset_core}/{print tolower(\$2)}'" - rset_whocares="awk '\$0 ~/^127\.0\.0\.1[ \t]+${rset_core}/{print tolower(\$2)}'" - rset_winhelp="awk '\$0 ~/^0\.0\.0\.0[ \t]+${rset_core}/{print tolower(\$2)}'" - rset_yoyo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - rset_zeus="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" - - # get logical wan update interfaces (with default route) and their device names - # - network_find_wan adb_wanif4 - network_find_wan6 adb_wanif6 - if [ -z "${adb_wanif4}" ] && [ -z "${adb_wanif6}" ] - then - rc=125 - f_log "no valid IPv4/IPv6 wan update interface found" "${rc}" - f_exit - elif [ "${adb_wanif4}" = "${adb_lanif}" ] || [ "${adb_wanif6}" = "${adb_lanif}" ] - then - rc=125 - f_log "LAN only (${adb_lanif}) network, no valid IPv4/IPv6 wan update interface found" "${rc}" - f_exit - else - network_get_device adb_wandev4 "${adb_wanif4}" - network_get_device adb_wandev6 "${adb_wanif6}" - fi - # get lan ip addresses # network_get_ipaddr adb_ipv4 "${adb_lanif}" network_get_ipaddr6 adb_ipv6 "${adb_lanif}" if [ -z "${adb_ipv4}" ] && [ -z "${adb_ipv6}" ] then - rc=130 - f_log "no valid IPv4/IPv6 configuration for given logical LAN interface found (${adb_lanif}), please set 'adb_lanif' manually" "${rc}" + rc=135 + f_log "no valid IPv4/IPv6 configuration found (${adb_lanif}), please set 'adb_lanif' manually" "${rc}" f_exit fi } -################################################# # f_envcheck: check/set environment prerequisites # f_envcheck() { local check + # check logical update interfaces (with default route) + # + network_find_wan adb_wanif4 + network_find_wan6 adb_wanif6 + if [ -z "${adb_wanif4}" ] && [ -z "${adb_wanif6}" ] + then + adb_wanif4="true" + f_log "no valid IPv4/IPv6 interface with default route found, IPv4 mode will be assumed" + fi + # check general package dependencies # f_depend "uhttpd" @@ -252,24 +184,13 @@ f_envcheck() fi fi - # check ca-certificates package and set wget parms accordingly + # check ca-certificates package and set fetch parms accordingly # - wget_parm="--no-config --quiet --tries=1 --no-cache --no-cookies --max-redirect=0 --dns-timeout=5 --connect-timeout=5 --read-timeout=5" + fetch_parm="--no-config --quiet --tries=1 --no-cache --no-cookies --max-redirect=0 --dns-timeout=5 --connect-timeout=5 --read-timeout=5" check="$(printf "${pkg_list}" | grep "^ca-certificates -")" if [ -z "${check}" ] then - wget_parm="${wget_parm} --no-check-certificate" - fi - - # check adblock blacklist/whitelist configuration - # - if [ ! -r "${adb_blacklist}" ] - then - f_log "adblock blacklist not found, source will be disabled" - fi - if [ ! -r "${adb_whitelist}" ] - then - f_log "adblock whitelist not found, source will be disabled" + fetch_parm="${fetch_parm} --no-check-certificate" fi # check adblock temp directory @@ -281,7 +202,7 @@ f_envcheck() then if [ $((av_space)) -le 2000 ] then - rc=135 + rc=140 f_log "not enough free space in '${adb_tmpdir}' (avail. ${av_space} kb)" "${rc}" f_exit else @@ -289,12 +210,12 @@ f_envcheck() fi fi else - rc=135 + rc=145 f_log "temp directory not found" "${rc}" f_exit fi - # memory check + # check memory # mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo")" mem_free="$(awk '$1 ~ /^MemFree/ {printf $2}' "/proc/meminfo")" @@ -309,12 +230,12 @@ f_envcheck() # check backup configuration # - if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ] + if [ -n "${adb_dir_backup}" ] && [ -d "${adb_dir_backup}" ] then - f_space "${adb_backupdir}" + f_space "${adb_dir_backup}" if [ "${space_ok}" = "false" ] then - f_log "not enough free space in '${adb_backupdir}'(avail. ${av_space} kb), backup/restore will be disabled" + f_log "not enough free space in '${adb_dir_backup}'(avail. ${av_space} kb), backup/restore will be disabled" backup_ok="false" else f_log "backup/restore will be enabled" @@ -325,17 +246,49 @@ f_envcheck() f_log "backup/restore will be disabled" fi + # check running dnsmasq instance & set defaults + # + rc="$(ps | grep -q "[d]nsmasq"; printf ${?})" + if [ $((rc)) -eq 0 ] + then + if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] + then + adb_dnsformat="awk -v ipv4="${adb_nullipv4}" -v ipv6="${adb_nullipv6}" '{print \"address=/\"\$0\"/\"ipv4\"\n\"\"address=/\"\$0\"/\"ipv6}'" + elif [ -n "${adb_wanif4}" ] + then + adb_dnsformat="awk -v ipv4="${adb_nullipv4}" '{print \"address=/\"\$0\"/\"ipv4}'" + else + adb_dnsformat="awk -v ipv6="${adb_nullipv6}" '{print \"address=/\"\$0\"/\"ipv6}'" + fi + else + rc=150 + f_log "please enable the local dns server to use adblock" "${rc}" + f_exit + fi + + # check running firewall + # + check="$(/usr/sbin/iptables -vnL | grep -F "DROP")" + if [ -z "${check}" ] + then + rc=155 + f_log "please enable the local firewall to use adblock" "${rc}" + f_exit + fi + # check ipv4/iptables configuration # - if [ -n "${adb_wanif4}" ] && [ -n "${adb_wandev4}" ] + if [ -n "${adb_wanif4}" ] then - f_firewall "IPv4" "nat" "A" "${adb_prechain_ipv4}" "adb-prerouting" "! -i ${adb_wandev4} -p tcp -d ${adb_nullipv4} -m multiport --dports 80,443 -j REDIRECT --to-ports ${adb_port}" - f_firewall "IPv4" "filter" "A" "${adb_fwdchain_ipv4}" "adb-forward" "! -i ${adb_wandev4} -d ${adb_nullipv4} -j REJECT --reject-with icmp-host-unreachable" - f_firewall "IPv4" "filter" "A" "${adb_outchain_ipv4}" "adb-output" "! -i ${adb_wandev4} -d ${adb_nullipv4} -j REJECT --reject-with icmp-host-unreachable" + f_firewall "IPv4" "nat" "A" "prerouting_rule" "adb-nat" "-p tcp -d ${adb_nullipv4} -m multiport --dports 80,443 -j DNAT --to-destination ${adb_ipv4}:${adb_nullport}" + f_firewall "IPv4" "filter" "A" "forwarding_rule" "adb-fwd" "-p tcp -d ${adb_nullipv4} -j REJECT --reject-with tcp-reset" + f_firewall "IPv4" "filter" "A" "forwarding_rule" "adb-fwd" "-d ${adb_nullipv4} -j REJECT --reject-with icmp-host-unreachable" + f_firewall "IPv4" "filter" "A" "output_rule" "adb-out" "-p tcp -d ${adb_nullipv4} -j REJECT --reject-with tcp-reset" + f_firewall "IPv4" "filter" "A" "output_rule" "adb-out" "-d ${adb_nullipv4} -j REJECT --reject-with icmp-host-unreachable" if [ $((adb_forcedns)) -eq 1 ] then - f_firewall "IPv4" "nat" "A" "${adb_prechain_ipv4}" "adb-dns" "! -i ${adb_wandev4} -p udp --dport 53 -j REDIRECT" - f_firewall "IPv4" "nat" "A" "${adb_prechain_ipv4}" "adb-dns" "! -i ${adb_wandev4} -p tcp --dport 53 -j REDIRECT" + f_firewall "IPv4" "nat" "A" "prerouting_rule" "adb-dns" "-p udp --dport 53 -j DNAT --to-destination ${adb_ipv4}:53" + f_firewall "IPv4" "nat" "A" "prerouting_rule" "adb-dns" "-p tcp --dport 53 -j DNAT --to-destination ${adb_ipv4}:53" fi if [ "${fw_done}" = "true" ] then @@ -346,15 +299,17 @@ f_envcheck() # check ipv6/ip6tables configuration # - if [ -n "${adb_wanif6}" ] && [ -n "${adb_wandev6}" ] + if [ -n "${adb_wanif6}" ] then - f_firewall "IPv6" "nat" "A" "${adb_prechain_ipv6}" "adb-prerouting" "! -i ${adb_wandev6} -p tcp -d ${adb_nullipv6} -m multiport --dports 80,443 -j REDIRECT --to-ports ${adb_port}" - f_firewall "IPv6" "filter" "A" "${adb_fwdchain_ipv6}" "adb-forward" "! -i ${adb_wandev6} -d ${adb_nullipv6} -j REJECT --reject-with icmp6-addr-unreachable" - f_firewall "IPv6" "filter" "A" "${adb_outchain_ipv6}" "adb-output" "! -i ${adb_wandev6} -d ${adb_nullipv6} -j REJECT --reject-with icmp6-addr-unreachable" + f_firewall "IPv6" "nat" "A" "PREROUTING" "adb-nat" "-p tcp -d ${adb_nullipv6} -m multiport --dports 80,443 -j DNAT --to-destination [${adb_ipv6}]:${adb_nullport}" + f_firewall "IPv6" "filter" "A" "forwarding_rule" "adb-fwd" "-p tcp -d ${adb_nullipv6} -j REJECT --reject-with tcp-reset" + f_firewall "IPv6" "filter" "A" "forwarding_rule" "adb-fwd" "-d ${adb_nullipv6} -j REJECT --reject-with icmp6-addr-unreachable" + f_firewall "IPv6" "filter" "A" "output_rule" "adb-out" "-p tcp -d ${adb_nullipv6} -j REJECT --reject-with tcp-reset" + f_firewall "IPv6" "filter" "A" "output_rule" "adb-out" "-d ${adb_nullipv6} -j REJECT --reject-with icmp6-addr-unreachable" if [ $((adb_forcedns)) -eq 1 ] then - f_firewall "IPv6" "nat" "A" "${adb_prechain_ipv6}" "adb-dns" "! -i ${adb_wandev6} -p udp --dport 53 -j REDIRECT" - f_firewall "IPv6" "nat" "A" "${adb_prechain_ipv6}" "adb-dns" "! -i ${adb_wandev6} -p tcp --dport 53 -j REDIRECT" + f_firewall "IPv6" "nat" "A" "PREROUTING" "adb-dns" "-p udp --dport 53 -j DNAT --to-destination [${adb_ipv6}]:53" + f_firewall "IPv6" "nat" "A" "PREROUTING" "adb-dns" "-p tcp --dport 53 -j DNAT --to-destination [${adb_ipv6}]:53" fi if [ "${fw_done}" = "true" ] then @@ -370,15 +325,14 @@ f_envcheck() then if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] then - uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/index.html" -p "${adb_ipv4}:${adb_port}" -p "[${adb_ipv6}]:${adb_port}" + uhttpd -h "/www/adblock" -k 0 -N 100 -t 0 -T 1 -D -S -E "/index.html" -p "${adb_ipv4}:${adb_nullport}" -p "[${adb_ipv6}]:${adb_nullport}" rc=${?} elif [ -n "${adb_wanif4}" ] then - uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/index.html" -p "${adb_ipv4}:${adb_port}" + uhttpd -h "/www/adblock" -k 0 -N 100 -t 0 -T 1 -D -S -E "/index.html" -p "${adb_ipv4}:${adb_nullport}" rc=${?} - elif [ -n "${adb_wanif6}" ] - then - uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/index.html" -p "[${adb_ipv6}]:${adb_port}" + else + uhttpd -h "/www/adblock" -k 0 -N 100 -t 0 -T 1 -D -S -E "/index.html" -p "[${adb_ipv6}]:${adb_nullport}" rc=${?} fi if [ $((rc)) -eq 0 ] @@ -390,17 +344,11 @@ f_envcheck() fi fi - # set dnsmasq defaults + # check whitelist entries # - if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] + if [ -s "${adb_whitelist}" ] then - adb_dnsformat="awk -v ipv4="${adb_nullipv4}" -v ipv6="${adb_nullipv6}" '{print \"address=/\"\$0\"/\"ipv4\"\n\"\"address=/\"\$0\"/\"ipv6}'" - elif [ -n "${adb_wanif4}" ] - then - adb_dnsformat="awk -v ipv4="${adb_nullipv4}" '{print \"address=/\"\$0\"/\"ipv4}'" - elif [ -n "${adb_wanif6}" ] - then - adb_dnsformat="awk -v ipv6="${adb_nullipv6}" '{print \"address=/\"\$0\"/\"ipv6}'" + awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist" fi # remove no longer used opkg package list @@ -408,7 +356,6 @@ f_envcheck() unset pkg_list } -###################################### # f_depend: check package dependencies # f_depend() @@ -419,13 +366,12 @@ f_depend() check="$(printf "${pkg_list}" | grep "^${package} -")" if [ -z "${check}" ] then - rc=140 + rc=160 f_log "package '${package}' not found" "${rc}" f_exit fi } -############################################## # f_firewall: set iptables rules for ipv4/ipv6 # f_firewall() @@ -461,12 +407,11 @@ f_firewall() fw_done="true" else f_log "failed to initialize volatile ${proto} firewall rule '${notes}'" "${rc}" - f_restore + f_exit fi fi } -########################################## # f_log: log messages to stdout and syslog # f_log() @@ -514,7 +459,49 @@ f_space() fi } -################################################################## +# f_cntconfig: calculate counters in config +# +f_cntconfig() +{ + local list + local src_name + local count=0 + local count_sum=0 + + for list in $(ls -ASr "${adb_dnsdir}/${adb_dnsprefix}."*) + do + src_name="${list/*./}" + count="$(wc -l < "${list}")" + if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] + then + count=$((count / 2)) + fi + uci_set "adblock" "${src_name}" "adb_src_count" "${count}" + count_sum=$((count_sum + count)) + done + uci_set "adblock" "global" "adb_overall_count" "${count_sum}" +} + +# f_rmconfig: remove counters & timestamps in given config sections +# +f_rmconfig() +{ + local rm_done="${1}" + + for list in ${rm_done} + do + src_name="${list/*./}" + if [ -n "${restore_done}" ] + then + uci_set "adblock" "${src_name}" "adb_src_timestamp" "list restored" + else + uci_remove "adblock" "${src_name}" "adb_src_count" + uci_remove "adblock" "${src_name}" "adb_src_timestamp" + fi + done + unset restore_done +} + # f_restore: restore last adblock list backups and restart dnsmasq # f_restore() @@ -530,6 +517,7 @@ f_restore() rc=${?} if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ] then + f_rmconfig "${rm_done}" f_log "all bogus adblock lists removed" elif [ $((rc)) -ne 0 ] then @@ -540,20 +528,20 @@ f_restore() # restore backups # - if [ "${backup_ok}" = "true" ] && [ "$(printf "${adb_backupdir}/${adb_dnsprefix}."*)" != "${adb_backupdir}/${adb_dnsprefix}.*" ] + if [ "${backup_ok}" = "true" ] then - restore_done="$(find "${adb_backupdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -print -exec cp -pf "{}" "${adb_dnsdir}" \;)" + restore_done="$(find "${adb_dir_backup}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -print -exec cp -pf "{}" "${adb_dnsdir}" \;)" rc=${?} if [ $((rc)) -eq 0 ] && [ -n "${restore_done}" ] then f_log "all available backups restored" - elif [ $((rc)) -ne 0 ] + elif [ $((rc)) -ne 0 ] && [ -n "${restore_done}" ] then f_log "error during restore of adblock lists" "${rc}" f_exit fi else - f_log "no backups found, nothing to restore" + f_log "backup service disabled, nothing to restore" fi # (re-)try dnsmasq restart without bogus adblock lists / with backups @@ -566,32 +554,22 @@ f_restore() if [ $((rc)) -eq 0 ] then rc=0 - adb_count="$(head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}."* | wc -l)" - if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] - then - adb_count="$((adb_count / 2))" - fi + f_cntconfig f_log "adblock lists with overall ${adb_count} domains loaded" else - rc=145 + rc=165 f_log "dnsmasq restart failed, please check 'logread' output" "${rc}" fi fi f_exit } -################################### -# f_exit: delete (temporary) files, -# generate statistics and exit +# f_exit: delete (temporary) files, generate statistics and exit # f_exit() { - local ipv4_prerouting=0 - local ipv4_forward=0 - local ipv4_output=0 - local ipv6_prerouting=0 - local ipv6_forward=0 - local ipv6_output=0 + local ipv4_adblock=0 + local ipv6_adblock=0 local iptv4="/usr/sbin/iptables" local iptv6="/usr/sbin/ip6tables" @@ -612,20 +590,19 @@ f_exit() then if [ -n "${adb_wanif4}" ] then - ipv4_prerouting="$(${iptv4} -t nat -vnL | awk '$11 ~ /^adb-prerouting$/ {sum += $1} END {printf sum}')" - ipv4_forward="$(${iptv4} -vnL | awk '$11 ~ /^adb-forward$/ {sum += $1} END {printf sum}')" - ipv4_output="$(${iptv4} -vnL | awk '$11 ~ /^adb-output$/ {sum += $1} END {printf sum}')" + ipv4_adblock="$(${iptv4} -t nat -vnL | awk '$11 ~ /^adb-nat$/ {sum += $1} END {printf sum}')" + ipv4_adblock="$((${ipv4_adblock} + $(${iptv4} -vnL | awk '$11 ~ /^adb-(fwd|out)$/ {sum += $1} END {printf sum}')))" fi if [ -n "${adb_wanif6}" ] then - ipv6_prerouting="$(${iptv6} -t nat -vnL | awk '$10 ~ /^adb-prerouting$/ {sum += $1} END {printf sum}')" - ipv6_forward="$(${iptv6} -vnL | awk '$10 ~ /^adb-forward$/ {sum += $1} END {printf sum}')" - ipv6_output="$(${iptv6} -vnL | awk '$10 ~ /^adb-output$/ {sum += $1} END {printf sum}')" + ipv6_adblock="$(${iptv6} -t nat -vnL | awk '$10 ~ /^adb-nat$/ {sum += $1} END {printf sum}')" + ipv6_adblock="$((${ipv6_adblock} + $(${iptv6} -vnL | awk '$10 ~ /^adb-(fwd|out)$/ {sum += $1} END {printf sum}')))" + fi + if [ -n "$(uci changes adblock)" ] + then + uci_commit "adblock" fi - f_log "adblock firewall statistics (IPv4/IPv6):" - f_log "${ipv4_prerouting}/${ipv6_prerouting} packets redirected in PREROUTING chain" - f_log "${ipv4_forward}/${ipv6_forward} packets rejected in FORWARD chain" - f_log "${ipv4_output}/${ipv6_output} packets rejected in OUTPUT chain" + f_log "firewall statistics (IPv4/IPv6): ${ipv4_adblock}/${ipv6_adblock} ad related packets blocked" f_log "domain adblock processing finished successfully (${adb_scriptver}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))" elif [ $((rc)) -gt 0 ] then diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh index cce8dcb172..54581b5f6d 100755 --- a/net/adblock/files/adblock-update.sh +++ b/net/adblock/files/adblock-update.sh @@ -36,7 +36,7 @@ fi # get current directory, script- and openwrt version # adb_scriptdir="${0%/*}" -adb_scriptver="1.0.3" +adb_scriptver="1.1.0" openwrt_version="$(cat /etc/openwrt_version)" # source in adblock function library @@ -63,102 +63,29 @@ f_envload # f_log "domain adblock processing started (${adb_scriptver}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))" -# parse environment -# -f_envparse - # check environment # f_envcheck -# start shallalist (pre-)processing -# -if [ -n "${adb_arc_shalla}" ] -then - # start shallalist processing - # - shalla_archive="${adb_tmpdir}/shallalist.tar.gz" - shalla_file="${adb_tmpdir}/shallalist.txt" - src_name="shalla" - adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}" - if [ -r "${adb_dnsfile}" ] - then - list_time="$(awk '$0 ~ /^# last modified/ {printf substr($0,18)}' "${adb_dnsfile}")" - fi - f_log "=> (pre-)processing adblock source '${src_name}'" - - # only process shallalist archive with updated timestamp, - # extract and merge only domains of selected shallalist categories - # - shalla_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${adb_arc_shalla}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')" - if [ -z "${shalla_time}" ] - then - shalla_time="$(date)" - f_log " no online timestamp received, current date will be used" - fi - if [ -z "${list_time}" ] || [ "${list_time}" != "${shalla_time}" ] - then - ${adb_fetch} ${wget_parm} --output-document="${shalla_archive}" "${adb_arc_shalla}" - rc=${?} - if [ $((rc)) -eq 0 ] - then - > "${shalla_file}" - for category in ${adb_cat_shalla} - do - tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}" - rc=${?} - if [ $((rc)) -ne 0 ] - then - f_log " archive extraction failed (${category})" - break - fi - done - - # remove temporary files - # - rm -f "${shalla_archive}" - rm -rf "${adb_tmpdir}/BL" - if [ $((rc)) -eq 0 ] - then - adb_sources="${adb_sources} ${shalla_file}&ruleset=rset_shalla" - f_log " source archive (pre-)processing finished" - else - rc=0 - adb_srclist="! -name ${adb_dnsprefix}.${src_name}" - adb_errsrclist="-name ${adb_dnsprefix}.${src_name}" - fi - else - rc=0 - adb_srclist="! -name ${adb_dnsprefix}.${src_name}" - adb_errsrclist="-name ${adb_dnsprefix}.${src_name}" - f_log " source archive download failed" - fi - else - adb_srclist="! -name ${adb_dnsprefix}.${src_name}" - f_log " source archive doesn't change, no update required" - fi -fi - -# add blacklist source to active adblock domain sources -# -if [ -s "${adb_blacklist}" ] -then - adb_sources="${adb_sources} ${adb_blacklist}&ruleset=rset_blacklist" -fi - # loop through active adblock domain sources, # download sources, prepare output and store all extracted domains in temp file # -for src in ${adb_sources} +for src_name in ${adb_sources} do - url="${src/\&ruleset=*/}" - src_name="${src/*\&ruleset=rset_/}" + eval "url=\"\${adb_src_${src_name}}\"" + eval "src_rset=\"\${adb_src_rset_${src_name}}\"" adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}" - if [ -r "${adb_dnsfile}" ] + list_time="$(uci_get "adblock" "${src_name}" "adb_src_timestamp")" + f_log "=> processing adblock source '${src_name}'" + + # check 'url' and 'src_rset' values + # + if [ -z "${url}" ] || [ -z "${src_rset}" ] then - list_time="$(awk '$0 ~ /^# last modified/ {printf substr($0,18)}' "${adb_dnsfile}")" + uci_set "adblock" "${src_name}" "adb_src_timestamp" "broken config" + f_log " broken source configuration, check 'adb_src' and 'adb_src_rset' in config" + continue fi - f_log "=> processing adblock source '${src_name}'" # prepare find statement with active adblock list sources # @@ -173,30 +100,48 @@ do # if [ "${src_name}" = "blacklist" ] then - url_time="$(date -r "${adb_blacklist}")" - elif [ "${src_name}" = "shalla" ] - then - url_time="${shalla_time}" + url_time="$(date -r "${url}")" else - url_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')" + url_time="$(${adb_fetch} ${fetch_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')" fi if [ -z "${url_time}" ] then url_time="$(date)" f_log " no online timestamp received, current date will be used" fi - if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ] + if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ] || [ ! -r "${adb_dnsfile}" ] ||\ + ([ "${backup_ok}" = "true" ] && [ ! -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}" ]) then if [ "${src_name}" = "blacklist" ] then - tmp_domains="$(cat "${adb_blacklist}")" + tmp_domains="$(cat "${url}")" rc=${?} elif [ "${src_name}" = "shalla" ] then - tmp_domains="$(cat "${shalla_file}")" + shalla_archive="${adb_tmpdir}/shallalist.tar.gz" + shalla_file="${adb_tmpdir}/shallalist.txt" + ${adb_fetch} ${fetch_parm} --output-document="${shalla_archive}" "${url}" rc=${?} + if [ $((rc)) -eq 0 ] + then + > "${shalla_file}" + for category in ${adb_src_cat_shalla} + do + tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}" + rc=${?} + if [ $((rc)) -ne 0 ] + then + f_log " archive extraction failed (${category})" + break + fi + done + rm -f "${shalla_archive}" + rm -rf "${adb_tmpdir}/BL" + tmp_domains="$(cat "${shalla_file}")" + rc=${?} + fi else - tmp_domains="$(${adb_fetch} ${wget_parm} --output-document=- "${url}")" + tmp_domains="$(${adb_fetch} ${fetch_parm} --output-document=- "${url}")" rc=${?} fi else @@ -208,8 +153,7 @@ do # if [ $((rc)) -eq 0 ] && [ -n "${tmp_domains}" ] then - eval "src_rset=\${rset_${src_name}}" - count="$(printf "%s\n" "${tmp_domains}" | eval "${src_rset}" | tee "${adb_tmpfile}" | wc -l)" + count="$(printf "%s\n" "${tmp_domains}" | awk "${src_rset}" | tee "${adb_tmpfile}" | wc -l)" f_log " source download finished (${count} entries)" if [ "${src_name}" = "shalla" ] then @@ -218,6 +162,7 @@ do unset tmp_domains elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ] then + uci_set "adblock" "${src_name}" "adb_src_timestamp" "empty download" f_log " empty source download finished" continue else @@ -228,6 +173,7 @@ do else adb_errsrclist="${adb_errsrclist} -o -name ${adb_dnsprefix}.${src_name}" fi + uci_set "adblock" "${src_name}" "adb_src_timestamp" "download failed" f_log " source download failed" continue fi @@ -237,9 +183,9 @@ do # if [ $((count)) -gt 0 ] && [ -n "${adb_tmpfile}" ] then - if [ -s "${adb_whitelist}" ] + if [ -s "${adb_tmpdir}/tmp.whitelist" ] then - grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}" + grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}" rc=${?} else sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}" @@ -255,19 +201,18 @@ do adb_revsrclist="${adb_revsrclist} -o -name ${adb_dnsprefix}.${src_name}" fi - # write preliminary footer + # store source timestamp in config # if [ $((rc)) -eq 0 ] then - printf "%s\n" "#---------------------------------------------" >> "${adb_dnsfile}" - printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}" - printf "%s\n" "##" >> "${adb_dnsfile}" + uci_set "adblock" "${src_name}" "adb_src_timestamp" "${url_time}" f_log " domain merging finished" else f_log " domain merging failed" "${rc}" f_restore fi else + uci_set "adblock" "${src_name}" "adb_src_timestamp" "empty domain input" f_log " empty domain input received" continue fi @@ -279,33 +224,23 @@ if [ -n "${adb_srclist}" ] then rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srclist} \) -print -exec rm -f "{}" \;)" rc=${?} + if [ "${backup_ok}" = "true" ] + then + find "${adb_dir_backup}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srclist} \) -exec rm -f "{}" \; + fi else rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -print -exec rm -f "{}" \;)" rc=${?} + if [ "${backup_ok}" = "true" ] + then + find "${adb_dir_backup}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -exec rm -f "{}" \; + fi fi if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ] then + f_rmconfig "${rm_done}" f_log "disabled adblock lists removed" - if [ "${backup_ok}" = "true" ] - then - if [ -n "${adb_srclist}" ] - then - rm_done="$(find "${adb_backupdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srclist} \) -print -exec rm -f "{}" \;)" - rc=${?} - else - rm_done="$(find "${adb_backupdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -print -exec rm -f "{}" \;)" - rc=${?} - fi - if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ] - then - f_log "disabled adblock list backups removed" - elif [ $((rc)) -ne 0 ] - then - f_log "error during removal of disabled adblock list backups" "${rc}" - f_exit - fi - fi -elif [ $((rc)) -ne 0 ] +elif [ $((rc)) -ne 0 ] && [ -n "${rm_done}" ] then f_log "error during removal of disabled adblock lists" "${rc}" f_exit @@ -315,10 +250,11 @@ fi # if [ "${backup_ok}" = "true" ] && [ -n "${adb_errsrclist}" ] then - restore_done="$(find "${adb_backupdir}" -maxdepth 1 -type f \( ${adb_errsrclist} \) -print -exec cp -pf "{}" "${adb_dnsdir}" \;)" + restore_done="$(find "${adb_dir_backup}" -maxdepth 1 -type f \( ${adb_errsrclist} \) -print -exec cp -pf "{}" "${adb_dnsdir}" \;)" rc=${?} if [ $((rc)) -eq 0 ] && [ -n "${restore_done}" ] then + f_rmconfig "${restore_done}" f_log "partial restore done" elif [ $((rc)) -ne 0 ] then @@ -335,52 +271,31 @@ then # generate a unique overall block list # - head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}."* | sort -u > "${adb_tmpdir}/blocklist.overall" + sort -u "${adb_dnsdir}/${adb_dnsprefix}."* > "${adb_tmpdir}/blocklist.overall" # loop through all separate lists, ordered by size (ascending) # - for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."*) + for list in $(ls -ASr "${adb_dnsdir}/${adb_dnsprefix}."*) do # check overall block list vs. separate block list, - # write only duplicate entries to a temporary separate list + # write all duplicate entries to separate list # list="${list/*./}" sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -d > "${adb_tmpdir}/tmp.${list}" + mv -f "${adb_tmpdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}" - # write only unique entries back to overall block list + # write all unique entries back to overall block list # - sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/tmp.${list}" | uniq -u > "${adb_tmpdir}/tmp.overall" + sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -u > "${adb_tmpdir}/tmp.overall" mv -f "${adb_tmpdir}/tmp.overall" "${adb_tmpdir}/blocklist.overall" - - # write unique result back to original separate list - # - tail -qn 3 "${adb_dnsdir}/${adb_dnsprefix}.${list}" >> "${adb_tmpdir}/tmp.${list}" - mv -f "${adb_tmpdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}" done rm -f "${adb_tmpdir}/blocklist.overall" fi -# set separate list count & get overall count -# -for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."*) -do - list="${list/*./}" - count="$(head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}.${list}" | wc -l)" - if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] - then - count=$((count / 2)) - fi - if [ "$(tail -qn 1 "${adb_dnsdir}/${adb_dnsprefix}.${list}")" = "##" ] - then - last_line="# ${0##*/} (${adb_scriptver}) - ${count} ad\/abuse domains blocked" - sed -i "s/^##$/${last_line}/" "${adb_dnsdir}/${adb_dnsprefix}.${list}" - fi - adb_count=$((adb_count + count)) -done - -# restart dnsmasq with newly generated or deleted adblock lists, -# check dnsmasq startup afterwards +# restart & check dnsmasq with newly generated set of adblock lists # +f_cntconfig +adb_count="$(uci_get "adblock" "global" "adb_overall_count")" if [ -n "${adb_revsrclist}" ] || [ -n "${rm_done}" ] || [ -n "${restore_done}" ] then /etc/init.d/dnsmasq restart @@ -395,19 +310,19 @@ then f_restore fi else - f_log "adblock lists with overall ${adb_count} domains are still valid, no dnsmasq restart required" + f_log "adblock lists with overall ${adb_count} domains are still valid, no update required" fi # create adblock list backups # -if [ "${backup_ok}" = "true" ] && [ -n "${adb_revsrclist}" ] && [ "$(printf "${adb_dnsdir}/${adb_dnsprefix}."*)" != "${adb_dnsdir}/${adb_dnsprefix}.*" ] +if [ "${backup_ok}" = "true" ] && [ -n "${adb_revsrclist}" ] then - backup_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec cp -pf "{}" "${adb_backupdir}" \;)" + backup_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec cp -pf "{}" "${adb_dir_backup}" \;)" rc=${?} if [ $((rc)) -eq 0 ] && [ -n "${backup_done}" ] then f_log "new adblock list backups generated" - elif [ $((rc)) -ne 0 ] + elif [ $((rc)) -ne 0 ] && [ -n "${backup_done}" ] then f_log "error during backup of adblock lists" "${rc}" f_exit diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index cb18787dcc..061f518129 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -3,96 +3,136 @@ config adblock 'global' option adb_enabled '1' - option adb_cfgver '1.0' - option adb_blacklist '/etc/adblock/adblock.blacklist' + option adb_cfgversion '1.1' option adb_whitelist '/etc/adblock/adblock.whitelist' + option adb_whitelist_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' option adb_forcedns '1' config service 'backup' option enabled '0' - option adb_backupdir '/mnt' + option adb_dir '/mnt' config source 'adaway' option enabled '1' - option adb_src 'https://adaway.org/hosts.txt&ruleset=rset_adaway' - option adb_srcdesc 'focus on mobile ads, infrequent updates, approx. 400 entries' + option adb_src 'https://adaway.org/hosts.txt' + option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' + option adb_src_desc 'focus on mobile ads, infrequent updates, approx. 400 entries' + +config source 'blacklist' + option enabled '0' + option adb_src '/etc/adblock/adblock.blacklist' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'static local domain blacklist (always deny these domains)' config source 'disconnect' option enabled '1' - option adb_src 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect' - option adb_srcdesc 'mozilla driven content blocklist, numerous updates on the same day, approx. 6.500 entries' + option adb_src 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'mozilla driven content blocklist, numerous updates on the same day, approx. 6.500 entries' config source 'dshield' option enabled '0' - option adb_src 'http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield' - option adb_srcdesc 'broad blocklist for suspicious domains, daily updates, approx. 4.500 entries' + option adb_src 'http://www.dshield.org/feeds/suspiciousdomains_Low.txt' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'broad blocklist for suspicious domains, daily updates, approx. 4.500 entries' config source 'feodo' option enabled '0' - option adb_src 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_feodo' - option adb_srcdesc 'focus on feodo botnet domains, daily updates, approx. 0-10 entries' + option adb_src 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'focus on feodo botnet domains, daily updates, approx. 0-10 entries' config source 'malware' option enabled '0' - option adb_src 'https://mirror.cedia.org.ec/malwaredomains/justdomains&ruleset=rset_malware' - option adb_srcdesc 'broad blocklist for malware domains, daily updates, approx. 16.000 entries' + option adb_src 'https://mirror.cedia.org.ec/malwaredomains/justdomains' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'broad blocklist for malware domains, daily updates, approx. 16.000 entries' config source 'malwarelist' option enabled '0' - option adb_src 'http://www.malwaredomainlist.com/hostslist/hosts.txt&ruleset=rset_malwarelist' - option adb_srcdesc 'generic blocklist for malware domains, daily updates, approx. 1.500 entries' + option adb_src 'http://www.malwaredomainlist.com/hostslist/hosts.txt' + option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' + option adb_src_desc 'generic blocklist for malware domains, daily updates, approx. 1.500 entries' config source 'openphish' option enabled '0' - option adb_src 'https://openphish.com/feed.txt&ruleset=rset_openphish' - option adb_srcdesc 'focus on phishing domains, numerous updates on the same day, approx. 1.800 entries' + option adb_src 'https://openphish.com/feed.txt' + option adb_src_rset '{FS=\"/\"} \$3 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$3)}' + option adb_src_desc 'focus on phishing domains, numerous updates on the same day, approx. 1.800 entries' config source 'palevo' option enabled '0' - option adb_src 'https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_palevo' - option adb_srcdesc 'focus on palevo worm domains, daily updates, approx. 15 entries' + option adb_src 'https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'focus on palevo worm domains, daily updates, approx. 15 entries' + +config source 'ransomware' + option enabled '0' + option adb_src 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'focus on ransomware domains, numerous updates on the same day, approx. 130 entries' + +config source 'rolist' + option enabled '0' + option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt' + option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' + option adb_src_desc 'focus on romanian ad related domains plus generic easylist additions, weekly updates, approx. 600 entries' config source 'ruadlist' option enabled '0' - option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt&ruleset=rset_ruadlist' - option adb_srcdesc 'focus on russian ad related domains plus generic easylist additions, weekly updates, approx. 2.000 entries' + option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt' + option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' + option adb_src_desc 'focus on russian ad related domains plus generic easylist additions, weekly updates, approx. 2.000 entries' config source 'shalla' option enabled '0' option adb_src 'http://www.shallalist.de/Downloads/shallalist.tar.gz' - option adb_srcdesc 'broad blocklist subdivided in different categories (adv, costtraps, spyware, tracker and warez enabled by default), daily updates, approx. 32.000 entries' - list adb_catlist 'adv' - list adb_catlist 'costtraps' - list adb_catlist 'spyware' - list adb_catlist 'tracker' - list adb_catlist 'warez' + option adb_src_rset '{FS=\"/\"} \$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'broad blocklist subdivided in different categories (adv, costtraps, spyware, tracker and warez enabled by default), daily updates, approx. 32.000 entries' + list adb_src_cat 'adv' + list adb_src_cat 'costtraps' + list adb_src_cat 'spyware' + list adb_src_cat 'tracker' + list adb_src_cat 'warez' config source 'spam404' option enabled '0' - option adb_src 'https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt&ruleset=rset_spam404' - option adb_srcdesc 'generic blocklist for suspicious domains, infrequent updates, approx. 5.000 entries' + option adb_src 'https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'generic blocklist for suspicious domains, infrequent updates, approx. 5.000 entries' config source 'sysctl' option enabled '0' - option adb_src 'http://sysctl.org/cameleon/hosts&ruleset=rset_sysctl' - option adb_srcdesc 'generic blocklist for ad related domains, weekly updates, approx. 21.000 entries' + option adb_src 'http://sysctl.org/cameleon/hosts' + option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' + option adb_src_desc 'generic blocklist for ad related domains, weekly updates, approx. 21.000 entries' config source 'whocares' option enabled '0' - option adb_src 'http://someonewhocares.org/hosts/hosts&ruleset=rset_whocares' - option adb_srcdesc 'broad blocklist for suspicious domains, weekly updates, approx. 12.000 entries' + option adb_src 'http://someonewhocares.org/hosts/hosts' + option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' + option adb_src_desc 'broad blocklist for suspicious domains, weekly updates, approx. 12.000 entries' + +config source 'winspy' + option enabled '0' + option adb_src 'https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/hostsBlockWindowsSpy.txt' + option adb_src_rset '\$0 ~/^0\.0\.0\.0[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' + option adb_src_desc 'focus on windows spy & telemetry domains, infrequent updates, approx. 120 entries' config source 'winhelp' option enabled '0' - option adb_src 'http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp' - option adb_srcdesc 'broad blocklist for suspicious domains, infrequent updates, approx. 15.000 entries' + option adb_src 'http://winhelp2002.mvps.org/hosts.txt' + option adb_src_rset '\$0 ~/^0\.0\.0\.0[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' + option adb_src_desc 'broad blocklist for suspicious domains, infrequent updates, approx. 15.000 entries' config source 'yoyo' option enabled '1' - option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext&ruleset=rset_yoyo' - option adb_srcdesc 'focus on ad related domains, weekly updates, approx. 2.500 entries' + option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'focus on ad related domains, weekly updates, approx. 2.500 entries' config source 'zeus' option enabled '0' - option adb_src 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus' - option adb_srcdesc 'focus on zeus botnet domains, daily updates, approx. 440 entries' + option adb_src 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist' + option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_desc 'focus on zeus botnet domains, daily updates, approx. 440 entries' diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init index 47b11dd729..f8b1262997 100755 --- a/net/adblock/files/adblock.init +++ b/net/adblock/files/adblock.init @@ -53,6 +53,10 @@ stop() rc=${?} if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ] then + . "/usr/bin/adblock-helper.sh" + uci_remove "adblock" "global" "adb_overall_count" + f_rmconfig "${rm_done}" + uci_commit "adblock" /etc/init.d/dnsmasq restart /etc/init.d/firewall restart fi diff --git a/net/adblock/files/www/adblock/adblock.png b/net/adblock/files/www/adblock/adblock.png deleted file mode 100644 index 9da19eacad3b03bb08bbddbbf4ac48dd78b3d838..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 68 zcmeAS@N?(olHy`uVBq!ia0vp^j3CUx0wlM}@Gt=>Zci7-kcv6Uzs@r-FtIZ-&5|)J Q1PU{Fy85}Sb4q9e0B4a5jsO4v diff --git a/net/adblock/files/www/adblock/index.html b/net/adblock/files/www/adblock/index.html index 29cdf586e8..2f8ba6e4e9 100644 --- a/net/adblock/files/www/adblock/index.html +++ b/net/adblock/files/www/adblock/index.html @@ -1,6 +1,5 @@ - - - + + -- 2.30.2