From: Dirk Brenken <dev@brenken.org>
Date: Mon, 19 Sep 2016 12:59:33 +0000 (+0300)
Subject: adblock: update to 1.4.9
X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=refs%2Fpull%2F3231%2Fhead;p=feed%2Fpackages.git

adblock: update to 1.4.9

Apply adblock updates 1.4.8-1.4.9

Signed-off-by: Hannu Nyman <hannu.nyman@iki.fi>

Original commit messages:

adblock: update 1.4.8
* enhance the new query function:
   change the regex to find only the relevant blocklist entries
   add a recursive tld search to quickly identify domains for
whitelisting (see documentation)
     better result preparation
* add securemecca as new blocklist source
* documentation update

adblock: update 1.4.9
* refine query function (see online doc)
---

diff --git a/net/adblock/Makefile b/net/adblock/Makefile
index d7061772e4..d79bf5199d 100644
--- a/net/adblock/Makefile
+++ b/net/adblock/Makefile
@@ -7,7 +7,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=1.4.7
+PKG_VERSION:=1.4.9
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>
@@ -23,7 +23,7 @@ endef
 
 define Package/$(PKG_NAME)/description
 Powerful adblock script to block ad/abuse domains.
-Currently the script supports 20 domain blacklist sites plus manual black- and whitelist overrides.
+The script supports 21 domain blacklist sites plus manual black- and whitelist overrides.
 Please see https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md for further information.
 
 endef
diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md
index 7ee7d1fbf9..e3b1972697 100644
--- a/net/adblock/files/README.md
+++ b/net/adblock/files/README.md
@@ -31,6 +31,8 @@ A lot of people already use adblocker plugins within their desktop browsers, but
     * => weekly updates, approx. 600 entries
     * [ruadlist/easylist](https://code.google.com/p/ruadlist)
     * => weekly updates, approx. 2.000 entries
+    * [securemecca](http://www.securemecca.com)
+    * => infrequent updates, approx. 25.000 entries
     * [shallalist](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default)
     * => daily updates, approx. 32.000 entries (a short description of all shallalist categories can be found [online](http://www.shallalist.de/categories.html))
     * [spam404](http://www.spam404.com)
@@ -63,6 +65,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but
 * init system support (start/stop/restart/reload/toggle/stats/cfgup)
 * hotplug support, the adblock start will be triggered by wan 'ifup' event, this can be restricted to a certain wan interface or disabled at all (see config options below)
 * toggle to quickly switch adblock 'on' or 'off'
+* query function to quickly identify blocked (sub-)domains, i.e. for whitelisting
 * optional: automatic adblock list backup/restore, backups will be (de-)compressed on the fly (disabled by default)
 * optional: add new adblock sources via uci config (see example below)
 
@@ -107,7 +110,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but
 * **restricted mode:** to disable flash writes with adblock status information to the adblock config file (used by LuCI frontend), please set 'adb\_restricted' to '1'
 * **adblock toggle:** to quickly switch adblocking 'on' or 'off', simply use _/etc/init.d/adblock toggle_
 * **adblock statistics:** to update only the adblock statistics (without updating the block lists as well), please run _/etc/init.d/adblock stats_
-* **adblock query <DOMAIN>:** to query the active blocklists for a specific domain, please run _/etc/init.d/adblock query <DOMAIN>_
+* **adblock query `<DOMAIN>`:** to query the active blocklists for a specific domain, please run _/etc/init.d/adblock query `<DOMAIN>`_ (see example below)
 * **configuration update:** to update an outdated adblock config file with the current default version, please run _/etc/init.d/adblock cfgup_, make your individual changes and start the adblock service again
 * **debugging:** for script debugging please set the 'adb\_debug' variable in the header of _/etc/init.d/adblock_ to '1'
 * **disable active dns probing in windows:** to prevent a possible yellow exclamation mark on your internet connection icon (which wrongly means connected, but no internet), please change the following registry key/value from "1" to "0" _HKLM\SYSTEM\CurrentControlSet\Services\NlaSvc\Parameters\Internet\EnableActiveProbing_
@@ -180,7 +183,28 @@ config uhttpd 'main'
     list listen_https '0.0.0.0:445'
 </code></pre>
   
-**example to find blocked domains on certain sites for whitelisting:**
+**example to query active blocklists for a certain (sub-)domain, i.e. for whitelisting:**
+<pre><code>
+/etc/init.d/adblock query "example.www.doubleclick.net"
+=> distinct results for domain 'example.www.doubleclick.net' (overall 0)
+   no matches in active blocklists
+=> distinct results for domain 'www.doubleclick.net' (overall 1)
+   adb_list.winhelp     : www.doubleclick.net
+=> distinct results for domain 'doubleclick.net' (overall 252)
+   adb_list.adaway      : ad-g.doubleclick.net
+   adb_list.hphosts     : 1016557.fls.doubleclick.net
+   adb_list.rolist      : feedads.g.doubleclick.net
+   adb_list.securemecca : 1168945.fls.doubleclick.net
+   adb_list.sysctl      : ad.co.doubleclick.net
+   adb_list.whocares    : 3ad.doubleclick.net
+   adb_list.winhelp     : 1435575.fls.doubleclick.net
+
+The query function checks against the submitted (sub-)domain and recurses automatically to the upper top level domain(s).
+For every domain it returns the overall count plus a distinct list of active blocklists with the first relevant result.
+In the example above you have to whitelist "www.doubleclick.net" to free the submitted domain.
+</code></pre>
+  
+**example to identify blocked domains during web browsing, i.e. for whitelisting:**
 <pre><code>
 1. the easy way ...
 enable the network analysis builtins in chrome or firefox to identify domains
diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh
index 1136d37f79..c1e467d7c2 100755
--- a/net/adblock/files/adblock-update.sh
+++ b/net/adblock/files/adblock-update.sh
@@ -10,8 +10,8 @@
 #
 adb_pid="${$}"
 adb_pidfile="/var/run/adblock.pid"
-adb_scriptver="1.4.7"
-adb_mincfgver="2.4"
+adb_scriptver="1.4.9"
+adb_mincfgver="2.5"
 adb_scriptdir="${0%/*}"
 if [ -r "${adb_pidfile}" ]
 then
diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf
index 401a0a5ff2..da58c2e833 100644
--- a/net/adblock/files/adblock.conf
+++ b/net/adblock/files/adblock.conf
@@ -3,7 +3,7 @@
 
 config adblock 'global'
 	option adb_enabled '1'
-	option adb_cfgver '2.4'
+	option adb_cfgver '2.5'
 	option adb_whitelist '/etc/adblock/adblock.whitelist'
 	option adb_whitelist_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}'
 	option adb_forcedns '1'
@@ -28,73 +28,79 @@ config source 'disconnect'
 	option enabled '1'
 	option adb_src 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt'
 	option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'mozilla driven content blocklist, numerous updates on the same day, approx. 6.500 entries'
+	option adb_src_desc 'mozilla driven blocklist, numerous updates on the same day, approx. 6.500 entries'
 
 config source 'dshield'
 	option enabled '0'
 	option adb_src 'http://www.dshield.org/feeds/suspiciousdomains_Low.txt'
 	option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'broad blocklist for suspicious domains, daily updates, approx. 4.500 entries'
+	option adb_src_desc 'generic blocklist, daily updates, approx. 4.500 entries'
 
 config source 'feodo'
 	option enabled '0'
 	option adb_src 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist'
 	option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'focus on feodo botnet domains, daily updates, approx. 0-10 entries'
+	option adb_src_desc 'focus on feodo botnet, daily updates, approx. 0-10 entries'
 
 config source 'hphosts'
 	option enabled '0'
 	option adb_src 'https://hosts-file.net/ad_servers.txt'
 	option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}'
-	option adb_src_desc 'broad blocklist against ad- and tracking hosts, monthly updates, approx. 50.000 entries'
+	option adb_src_desc 'broad blocklist, monthly updates, approx. 50.000 entries'
 
 config source 'malware'
 	option enabled '0'
 	option adb_src 'https://mirror.cedia.org.ec/malwaredomains/justdomains'
 	option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'broad blocklist for malware domains, daily updates, approx. 16.000 entries'
+	option adb_src_desc 'broad blocklist, daily updates, approx. 16.000 entries'
 
 config source 'malwarelist'
 	option enabled '0'
 	option adb_src 'http://www.malwaredomainlist.com/hostslist/hosts.txt'
 	option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}'
-	option adb_src_desc 'generic blocklist for malware domains, daily updates, approx. 1.500 entries'
+	option adb_src_desc 'focus on malware, daily updates, approx. 1.500 entries'
 
 config source 'openphish'
 	option enabled '0'
 	option adb_src 'https://openphish.com/feed.txt'
 	option adb_src_rset '{FS=\"/\"} \$3 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$3)}'
-	option adb_src_desc 'focus on phishing domains, numerous updates on the same day, approx. 1.800 entries'
+	option adb_src_desc 'focus on phishing, numerous updates on the same day, approx. 1.800 entries'
 
 config source 'palevo'
 	option enabled '0'
 	option adb_src 'https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist'
 	option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'focus on palevo worm domains, daily updates, approx. 15 entries'
+	option adb_src_desc 'focus on palevo worm, daily updates, approx. 15 entries'
 
 config source 'ransomware'
 	option enabled '0'
 	option adb_src 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt'
 	option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'focus on ransomware domains, numerous updates on the same day, approx. 130 entries'
+	option adb_src_desc 'focus on ransomware, numerous updates on the same day, approx. 130 entries'
 
 config source 'rolist'
 	option enabled '0'
 	option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt'
 	option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}'
-	option adb_src_desc 'focus on romanian ad related domains plus generic easylist additions, weekly updates, approx. 600 entries'
+	option adb_src_desc 'focus on romanian ads plus generic easylist additions, weekly updates, approx. 600 entries'
 
 config source 'ruadlist'
 	option enabled '0'
 	option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt'
 	option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}'
-	option adb_src_desc 'focus on russian ad related domains plus generic easylist additions, weekly updates, approx. 2.000 entries'
+	option adb_src_desc 'focus on russian ads plus generic easylist additions, weekly updates, approx. 2.000 entries'
+
+config source 'securemecca'
+    option enabled '0'
+    option adb_src 'http://securemecca.com/Downloads/hosts.txt'
+    option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}'
+    option adb_src_desc 'broad blocklist, infrequent updates, approx. 25.000 entries'
 
 config source 'shalla'
 	option enabled '0'
 	option adb_src 'http://www.shallalist.de/Downloads/shallalist.tar.gz'
 	option adb_src_rset '{FS=\"/\"} \$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'broad blocklist subdivided in different categories (adv, costtraps, spyware, tracker and warez enabled by default), daily updates, approx. 32.000 entries'
+	option adb_src_desc 'broad blocklist subdivided in different categories, daily updates, approx. 32.000 entries'
 	list adb_src_cat 'adv'
 	list adb_src_cat 'costtraps'
 	list adb_src_cat 'spyware'
@@ -105,31 +111,31 @@ config source 'spam404'
 	option enabled '0'
 	option adb_src 'https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt'
 	option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'generic blocklist for suspicious domains, infrequent updates, approx. 5.000 entries'
+	option adb_src_desc 'generic blocklist, infrequent updates, approx. 5.000 entries'
 
 config source 'sysctl' 
 	option enabled '0'
 	option adb_src 'http://sysctl.org/cameleon/hosts'
 	option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}'
-	option adb_src_desc 'generic blocklist for ad related domains, weekly updates, approx. 21.000 entries'
+	option adb_src_desc 'broad blocklist, weekly updates, approx. 21.000 entries'
 
 config source 'whocares'
 	option enabled '0'
 	option adb_src 'http://someonewhocares.org/hosts/hosts'
 	option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}'
-	option adb_src_desc 'broad blocklist for suspicious domains, weekly updates, approx. 12.000 entries'
+	option adb_src_desc 'broad blocklist, weekly updates, approx. 12.000 entries'
 
 config source 'winspy'
 	option enabled '0'
 	option adb_src 'https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/win10/spy.txt'
 	option adb_src_rset '\$0 ~/^0\.0\.0\.0[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}'
-	option adb_src_desc 'focus on windows spy & telemetry domains, infrequent updates, approx. 120 entries'
+	option adb_src_desc 'focus on windows spy & telemetry, infrequent updates, approx. 140 entries'
 
 config source 'winhelp'
 	option enabled '0'
 	option adb_src 'http://winhelp2002.mvps.org/hosts.txt'
 	option adb_src_rset '\$0 ~/^0\.0\.0\.0[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}'
-	option adb_src_desc 'broad blocklist for suspicious domains, infrequent updates, approx. 15.000 entries'
+	option adb_src_desc 'broad blocklist, infrequent updates, approx. 15.000 entries'
 
 config source 'yoyo'
 	option enabled '1'
@@ -141,4 +147,4 @@ config source 'zeus'
 	option enabled '0'
 	option adb_src 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist'
 	option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}'
-	option adb_src_desc 'focus on zeus botnet domains, daily updates, approx. 440 entries'
+	option adb_src_desc 'focus on zeus botnet, daily updates, approx. 440 entries'
diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init
index b0dc777d04..f7b707a21b 100755
--- a/net/adblock/files/adblock.init
+++ b/net/adblock/files/adblock.init
@@ -131,12 +131,31 @@ cfgup()
 
 query()
 {
-    if [ -z "${1}" ]
+    domain="${1}"
+    tld="${domain#*.}"
+    list_dns="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)"
+    if [ -z "${list_dns}" ]
     then
-        f_log "no domain query input, please submit a specific search domain"
+        f_log "no active blocklists found, please start adblock first"
+    elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ]
+    then
+        f_log "invalid domain query input, please submit a specific (sub-)domain, i.e. 'www.abc.xyz'"
     else
-        f_log "results for search domain '${1}':"
-        grep "${1}" "/tmp/dnsmasq.d/adb_list"*
+        while [ "${domain}" != "${tld}" ]
+        do
+            search="${domain//./\.}"
+            result="$(grep -Hm 1 "[/\.]${search}/" "${adb_dnsdir}/adb_list"* | awk -F ':|/' '{print "   "$4"\t: "$6}')"
+            count="$(grep -hc "[/\.]${search}/" "${adb_dnsdir}/adb_list"* | awk '{sum += $1} END {printf sum}')"
+            printf "%s\n" "=> distinct results for domain '${domain}' (overall ${count})"
+            if [ -z "${result}" ]
+            then
+                printf "%s\n" "   no matches in active blocklists"
+            else
+                printf "%s\n" "${result}"
+            fi
+            domain="${tld}"
+            tld="${domain#*.}"
+        done
     fi
     return 0
 }