From e660813798832ffd8d7aad0c53161318c93eb024 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Mon, 26 Jun 2017 20:58:29 +0200 Subject: [PATCH] adblock: update 2.8.0 * add bind support (see readme) * export all blocked domains in one central file (adb_list.overall) * prerequisite for proper bind support * much faster sort operation with less memory consumption * backups are still handled per source separately, to be more flexible in adding/removing block list sources * add additional 'wan6' interface trigger in default configuration * various small fixes & optimizations Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 4 +- net/adblock/files/README.md | 77 +++++++++++++++--------- net/adblock/files/adblock.conf | 2 +- net/adblock/files/adblock.sh | 104 +++++++++++++++------------------ 4 files changed, 100 insertions(+), 87 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index e7bb701bb1..c1fd658d28 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=2.7.1 +PKG_VERSION:=2.8.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken @@ -21,7 +21,7 @@ define Package/adblock endef define Package/adblock/description -Powerful adblock script to block ad/abuse domains via dnsmasq or unbound dns backend. +Powerful adblock script to block ad/abuse domains via dnsmasq, unbound or bind dns backend. The script supports many domain blacklist sites plus manual black- and whitelist overrides. Please see https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md for further information. diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 464a90a2f5..0758d7fe2f 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -55,15 +55,15 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => daily updates, approx. 440 entries * zero-conf like automatic installation & setup, usually no manual changes needed * simple but yet powerful adblock engine: adblock does not use error prone external iptables rulesets, http pixel server instances and things like that -* automatically selects dnsmasq or unbound as dns backend +* automatically selects dnsmasq, unbound or bind as dns backend * automatically selects uclient-fetch or wget as download utility (other tools like curl or aria2c are supported as well) * support http only mode (without installed ssl library) for all non-SSL blocklist sources * automatically supports a wide range of router modes, even AP modes are supported * full IPv4 and IPv6 support -* supports tld compression (top level domain compression), this feature removes thousands of needless host entries from the block lists and lowers the memory footprint for the dns backends +* supports tld compression (top level domain compression), this feature removes thousands of needless host entries from the block list and lowers the memory footprint for the dns backends * each block list source will be updated and processed separately * block list source parsing by fast & flexible regex rulesets -* overall duplicate removal in separate block lists +* overall duplicate removal in central block list (adb_list.overall) * additional whitelist for manual overrides, located by default in /etc/adblock/adblock.whitelist * quality checks during block list update to ensure a reliable dns backend service * minimal status & error logging to syslog, enable debug logging to receive more output @@ -105,7 +105,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * **scheduled list updates:** for a scheduled call of the adblock service add an appropriate crontab entry (see example below) * **restrict procd interface trigger:** restrict the procd interface trigger to a (list of) certain interface(s) (default: wan). To disable it at all, remove all entries * **suspend & resume adblocking:** to quickly switch the adblock service 'on' or 'off', simply use _/etc/init.d/adblock [suspend|resume]_ -* **domain query:** to query the active block lists for a specific domain, please run _/etc/init.d/adblock query ``_ (see example below) +* **domain query:** to query the active block list for a specific domain, please run _/etc/init.d/adblock query ``_ (see example below) * **add new list sources:** you could add new block list sources on your own via uci config, all you need is a source url and an awk one-liner (see example below) * **disable active dns probing in windows 10:** to prevent a yellow exclamation mark on your internet connection icon (which wrongly means connected, but no internet), please change the following registry key/value from "1" to "0" _HKLM\SYSTEM\CurrentControlSet\Services\NlaSvc\Parameters\Internet\EnableActiveProbing_ @@ -119,16 +119,40 @@ A lot of people already use adblocker plugins within their desktop browsers, but * adb\_triggerdelay => additional trigger delay in seconds before adblock processing starts (default: '2') * adb\_forcedns => force dns requests to local resolver (default: '0', disabled) * adb\_forcesrt => force overall sort on low memory devices with less than 64 MB RAM (default: '0', disabled) - * adb\_manmode => do not automatically update blocklists during startup, use blocklist backups instead (default: '0', disabled) + * adb\_manmode => do not automatically update block lists during startup, use backups instead (default: '0', disabled) ## Examples **change default dns backend to 'unbound':** 

-Adblock detects the presence of an active unbound dns backend and the block lists will be automatically pulled in by unbound.
-The adblock script deposits the sorted and filtered block lists in '/var/lib/unbound' where unbound can find them in its jail.
-If you use manual configuration for unbound, then just include the following line in your 'server:' clause:
+Adblock deposits the sorted and filtered block list (adb_list.overall) in '/var/lib/unbound' where unbound can find them in its jail.
+If you use manual configuration for unbound, then just include the following line in your 'server' clause:
 
-  include: "/var/lib/unbound/adb_list.*"
+  include: "/var/lib/unbound/adb_list.overall"
+
+ +**change default dns backend to 'bind':** +

+Adblock deposits the sorted and filtered block list (adb_list.overall) in '/var/lib/bind' where bind can find them.
+To use the block list please modify the following bind configuration files:
+
+change '/etc/bind/named.conf', in the 'options' namespace add:
+  response-policy { zone "rpz"; };
+
+and at the end of the file add:
+  zone "rpz" {
+    type master;
+    file "/etc/bind/db.rpz";
+    allow-query { none; };
+    allow-transfer { none; };
+  };
+
+create the new file '/etc/bind/db.rpz' and add:
+  $TTL 2h
+  $ORIGIN rpz.
+  @ SOA localhost. root.localhost. (1 6h 1h 1w 2h)
+  NS localhost.
+
+  $INCLUDE /var/lib/bind/adb_list.overall
**configuration for different download utilities:** @@ -152,15 +176,15 @@ curl: **receive adblock runtime information:** 

-root@blackhole:~# /etc/init.d/adblock status
+/etc/init.d/adblock status
 ::: adblock runtime information
  status          : active
- adblock_version : 2.6.0
- blocked_domains : 113711
+ adblock_version : 2.8.0
+ blocked_domains : 122827
  fetch_info      : wget (built-in)
  dns_backend     : dnsmasq
- last_rundate    : 12.04.2017 13:08:26
- system          : LEDE Reboot SNAPSHOT r3900-399d5cf532
+ last_rundate    : 26.06.2017 17:00:27
+ system          : LEDE Reboot SNAPSHOT r4434-b91a38d647
**cronjob for a regular block list update (/etc/crontabs/root):** @@ -186,7 +210,7 @@ This entry does not block: 

 here.com
 
-This entry removes the following (sub)domains from the block lists:
+This entry removes the following (sub)domains from the block list:
   maps.here.com
   here.com
 
@@ -195,22 +219,21 @@ This entry does not remove:
   www.adwhere.com
-**query active block lists for a certain (sub-)domain, e.g. for whitelisting:** +**query active block list for a certain (sub-)domain, e.g. for whitelisting:** 

 /etc/init.d/adblock query example.www.doubleclick.net
-::: distinct results for domain 'example.www.doubleclick.net'
- no match
-::: distinct results for domain 'www.doubleclick.net'
- adb_list.sysctl      : www.doubleclick.net
-::: distinct results for domain 'doubleclick.net'
- adb_list.adaway      : ad-g.doubleclick.net
- adb_list.securemecca : 1168945.fls.doubleclick.net
- adb_list.sysctl      : 1435575.fls.doubleclick.net
- adb_list.whocares    : 3ad.doubleclick.net
+::: results for (sub-)domain 'example.www.doubleclick.net' (max. 5)
+ - no match
+::: results for (sub-)domain 'www.doubleclick.net' (max. 5)
+ - no match
+::: results for (sub-)domain 'doubleclick.net' (max. 5)
+ + doubleclick.net
+ + feedads.g.doubleclick.net
+ + survey.g.doubleclick.net
 
 The query function checks against the submitted (sub-)domain and recurses automatically to the upper top level domain(s).
-For every domain it returns the overall count plus a distinct list of active block lists with the first relevant result.
-In the example above whitelist "www.doubleclick.net" to free the submitted domain.
+For every (sub-)domain it returns the first five relevant results.
+In the example above whitelist "doubleclick.net" to free the submitted domain.
**add a new block list source:** diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index c26fb9e273..6386f40b6b 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -6,7 +6,7 @@ config adblock 'global' option adb_debug '0' option adb_forcesrt '0' option adb_forcedns '0' - option adb_iface 'wan' + option adb_iface 'wan wan6' option adb_triggerdelay '2' option adb_whitelist '/etc/adblock/adblock.whitelist' option adb_whitelist_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}' diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index 41c1be737c..028f421101 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -10,7 +10,7 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="2.7.1" +adb_ver="2.8.0" adb_sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')" adb_enabled=1 adb_debug=0 @@ -24,8 +24,9 @@ adb_whitelist="/etc/adblock/adblock.whitelist" adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}" adb_fetch="/usr/bin/wget" adb_fetchparm="--quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O" -adb_dnslist="dnsmasq unbound" +adb_dnslist="dnsmasq unbound named" adb_dnsprefix="adb_list" +adb_dnsfile="${adb_dnsprefix}.overall" adb_rtfile="/tmp/adb_runtime.json" adb_sources="" adb_src_cat_shalla="" @@ -113,6 +114,13 @@ f_envload() adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'" break 2 ;; + named) + adb_dns="${dns}" + adb_dnsdir="${adb_dnsdir:="/var/lib/bind"}" + adb_dnshidedir="${adb_dnsdir}/.adb_hidden" + adb_dnsformat="awk '{print \"\"\$0\" IN CNAME .\n*.\"\$0\" IN CNAME .\"}'" + break 2 + ;; esac fi done @@ -160,7 +168,7 @@ f_envcheck() # if [ ${adb_enabled} -ne 1 ] then - if [ -n "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null)" ] + if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] then f_rmdns f_dnsrestart @@ -291,18 +299,14 @@ f_list() fi ;; restore) - if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ] + if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ] && + [ -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ] then - rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" - if [ -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ] - then - gunzip -cf "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}" - adb_rc=${?} - fi + gunzip -cf "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}" + adb_rc=${?} fi ;; remove) - rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" if [ -d "${adb_backupdir}" ] then rm -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" @@ -312,9 +316,9 @@ f_list() format) if [ -s "${adb_tmpdir}/tmp.whitelist" ] then - grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}" + grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | eval "${adb_dnsformat}" >> "${adb_tmpdir}/${adb_dnsfile}" else - eval "${adb_dnsformat}" "${adb_tmpfile}" > "${adb_dnsfile}" + eval "${adb_dnsformat}" "${adb_tmpfile}" >> "${adb_tmpdir}/${adb_dnsfile}" fi adb_rc=${?} ;; @@ -326,20 +330,18 @@ f_list() # f_switch() { + local source target status mode="${1}" + if [ -d "${adb_dnshidedir}" ] then - local source target status mode="${1}" - local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" - local dns_passive="$(find "${adb_dnshidedir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" - - if [ -n "${dns_active}" ] && [ "${mode}" = "suspend" ] + if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] && [ "${mode}" = "suspend" ] then - source="${adb_dnsdir}/${adb_dnsprefix}" + source="${adb_dnsdir}/${adb_dnsfile}" target="${adb_dnshidedir}" status="suspended" - elif [ -n "${dns_passive}" ] && [ "${mode}" = "resume" ] + elif [ -s "${adb_dnshidedir}/${adb_dnsfile}" ] && [ "${mode}" = "resume" ] then - source="${adb_dnshidedir}/${adb_dnsprefix}" + source="${adb_dnshidedir}/${adb_dnsfile}" target="${adb_dnsdir}" status="resumed" fi @@ -352,18 +354,17 @@ f_switch() fi } -# f_query: query block lists for certain (sub-)domains +# f_query: query block list for certain (sub-)domains # f_query() { local search result cnt local domain="${1}" local tld="${domain#*.}" - local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" - if [ -z "${dns_active}" ] + if [ ! -s "${adb_dnsdir}/${adb_dnsfile}" ] then - printf "%s\n" "::: no active block lists found, please start / resume adblock first" + printf "%s\n" "::: no active block list found, please start / resume adblock first" elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] then printf "%s\n" "::: invalid domain input, please submit a specific (sub-)domain, e.g. 'www.abc.xyz'" @@ -372,9 +373,9 @@ f_query() while [ "${domain}" != "${tld}" ] do search="${domain//./\.}" - result="$(grep -Hm1 "[/\"\.]${search}[/\"]" "${adb_dnsprefix}"* | awk -F ':|=|/|\"' '{printf(" %-20s : %s\n",$1,$4)}')" - printf "%s\n" "::: distinct results for domain '${domain}'" - printf "%s\n" "${result:=" no match"}" + result="$(grep -Hm5 "[/\"\.]${search}[/\"]" "${adb_dnsfile}" | awk -F ':|=|/|\"' '{printf(" + %s\n",$4)}')" + printf "%s\n" "::: results for (sub-)domain '${domain}' (max. 5)" + printf "%s\n" "${result:=" - no match"}" domain="${tld}" tld="${domain#*.}" done @@ -389,14 +390,10 @@ f_status() if [ -s "${adb_rtfile}" ] then - local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" - local dns_passive="$(find "${adb_dnshidedir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" - - if [ -n "${dns_active}" ] + if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] then value="active" - elif [ -n "${dns_passive}" ] || [ -z "${dns_active}" ] - then + else value="no domains blocked" fi printf "%s\n" "::: adblock runtime information" @@ -425,7 +422,7 @@ f_log() then logger -t "adblock-[${adb_ver}] ${class}" "Please check 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md' (${adb_sysver})" f_rmtemp - if [ -n "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null)" ] + if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] then f_rmdns f_dnsrestart @@ -439,7 +436,7 @@ f_log() # f_main() { - local src_name src_rset shalla_archive enabled url cnt sum_cnt=0 + local src_name src_rset shalla_archive enabled url cnt=0 local mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo")" f_log "info " "start adblock processing ..." @@ -450,10 +447,9 @@ f_main() eval "enabled=\"\${enabled_${src_name}}\"" eval "url=\"\${adb_src_${src_name}}\"" eval "src_rset=\"\${adb_src_rset_${src_name}}\"" - adb_dnsfile="${adb_tmpdir}/${adb_dnsprefix}.${src_name}" > "${adb_tmpload}" > "${adb_tmpfile}" - adb_rc=0 + adb_rc=4 # basic pre-checks # @@ -468,7 +464,6 @@ f_main() # if [ ${adb_manmode} -eq 1 ] && [ -z "${adb_action}" ] then - adb_rc=4 f_list restore if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] then @@ -479,7 +474,7 @@ f_main() # download block list # - if [ "${src_name}" = "blacklist" ] + if [ "${src_name}" = "blacklist" ] && [ -s "${url}" ] then cat "${url}" > "${adb_tmpload}" adb_rc=${?} @@ -542,25 +537,20 @@ f_main() # overall sort # - for src_name in $(ls -dASr "${adb_tmpdir}/${adb_dnsprefix}"* 2>/dev/null) - do - if [ ${mem_total} -ge 64 ] || [ ${adb_forcesrt} -eq 1 ] + if [ ${mem_total} -ge 64 ] || [ ${adb_forcesrt} -eq 1 ] + then + if [ -s "${adb_tmpdir}/${adb_dnsfile}" ] then - if [ -s "${adb_tmpdir}/blocklist.overall" ] - then - sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/blocklist.overall" "${src_name}" | uniq -u > "${adb_tmpdir}/tmp.blocklist" - mv -f "${adb_tmpdir}/tmp.blocklist" "${src_name}" - fi - cat "${src_name}" >> "${adb_tmpdir}/blocklist.overall" + sort -u "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" + else + mv -f "${adb_tmpdir}/${adb_dnsfile}" "${adb_dnsdir}" 2>/dev/null fi - cnt="$(wc -l < "${src_name}")" - sum_cnt=$((sum_cnt + cnt)) - done + cnt="$(wc -l < "${adb_dnsdir}/${adb_dnsfile}")" + fi # restart the dns backend and export runtime information # - mv -f "${adb_tmpdir}/${adb_dnsprefix}"* "${adb_dnsdir}" 2>/dev/null - chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null + chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null f_rmtemp f_dnsrestart if [ ${?} -eq 0 ] @@ -568,16 +558,16 @@ f_main() json_init json_add_object "data" json_add_string "adblock_version" "${adb_ver}" - json_add_string "blocked_domains" "${sum_cnt}" + json_add_string "blocked_domains" "${cnt}" json_add_string "fetch_info" "${adb_fetchinfo}" json_add_string "dns_backend" "${adb_dns}" json_add_string "last_rundate" "$(/bin/date "+%d.%m.%Y %H:%M:%S")" json_add_string "system" "${adb_sysver}" json_close_object json_dump > "${adb_rtfile}" - f_log "info " "block lists with overall ${sum_cnt} domains loaded successfully (${adb_sysver})" + f_log "info " "block list with overall ${cnt} domains loaded successfully (${adb_sysver})" else - f_log "error" "dns backend restart with active block lists failed" + f_log "error" "dns backend restart with active block list failed" fi } -- 2.30.2