From afe6be45e3c55343b61f538986a3c62b97dd2984 Mon Sep 17 00:00:00 2001
From: Dirk Brenken <dev@brenken.org>
Date: Sat, 4 Mar 2017 18:20:24 +0200
Subject: [PATCH] adblock: backport updates upto 2.4.0-2

Signed-off-by: Hannu Nyman <hannu.nyman@iki.fi>

Original commit messages:

adblock: update 2.3.2
* optimize memory consumption &
  enable overall sort only on devices with > 64MB RAM,
  this prevents sort related kernel dumps
(cherry picked from commit 8c5b9a0802dec0876488779f0836c5b4698388cc)

adblock: release 2.4.0
* add tld compression,
  this new "top level domain compression" removes up to 40 thousand
  needless host entries from the block lists and
  lowers the memory footprint for the dns backends by 8-10 MByte
* optimize restart behavior in case of an error
* cosmetics
(cherry picked from commit ed470f0dcc66f42bc57e3795e3c9f37629e2cbcd)

adblock: release 2.4.0 (release 2)
* add missing sort step if tld compression was disabled
(cherry picked from commit b3b9972eacdf4acc5ff231f7aa5c32d14fbc4841)
---
 net/adblock/Makefile           |   4 +-
 net/adblock/files/README.md    |  10 +--
 net/adblock/files/adblock.init |   7 +-
 net/adblock/files/adblock.sh   | 118 ++++++++++++++++++---------------
 4 files changed, 75 insertions(+), 64 deletions(-)

diff --git a/net/adblock/Makefile b/net/adblock/Makefile
index a10fcd4c82..9c7f746b8f 100644
--- a/net/adblock/Makefile
+++ b/net/adblock/Makefile
@@ -6,8 +6,8 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=2.3.1
-PKG_RELEASE:=1
+PKG_VERSION:=2.4.0
+PKG_RELEASE:=2
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>
 
diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md
index 62732e9075..172551f5cd 100644
--- a/net/adblock/files/README.md
+++ b/net/adblock/files/README.md
@@ -57,6 +57,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but
 * automatically selects uclient-fetch or wget as download utility (other tools like curl or aria2c are supported as well)
 * automatically supports a wide range of router modes, even AP modes are supported
 * full IPv4 and IPv6 support
+* supports tld compression (top level domain compression), this feature removes thousands of needless host entries from the block lists and lowers the memory footprint for the dns backends
 * each block list source will be updated and processed separately
 * block list source parsing by fast & flexible regex rulesets
 * overall duplicate removal in separate block lists
@@ -120,6 +121,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but
     * adb\_iface => restrict the procd interface trigger to a (list of) certain wan interface(s) or disable it at all (default: not set, disabled)
     * adb\_fetch => full path to a different download utility, see example below (default: not set, use wget)
     * adb\_fetchparm => options for the download utility, see example below (default: not set, use wget options)
+    * adb\_tldcomp => enable/disable tld compression (default: '1', enabled)
 
 ## Examples
 
@@ -136,19 +138,19 @@ If you use manual configuration for unbound, then just include the following lin
 <pre><code>
 wget (default):
   option adb_fetch="/usr/bin/wget"
-  option adb_fetchparm="--no-config --quiet --tries=1 --no-cache --no-cookies --max-redirect=0 --timeout=5 --no-check-certificate -O"
+  option adb_fetchparm="--no-config --quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O"
 
 aria2c:
   option adb_fetch '/usr/bin/aria2c'
-  option adb_fetchparm '-q --max-tries=1 --timeout=5 --allow-overwrite=true --auto-file-renaming=false --check-certificate=false -o'
+  option adb_fetchparm '-q --timeout=10 --allow-overwrite=true --auto-file-renaming=false --check-certificate=false -o'
 
 uclient-fetch:
   option adb_fetch '/bin/uclient-fetch'
-  option adb_fetchparm '-q --timeout=5 --no-check-certificate -O'
+  option adb_fetchparm '-q --timeout=10 --no-check-certificate -O'
 
 curl:
   option adb_fetch '/usr/bin/curl'
-  option adb_fetchparm '-s --retry 1 --connect-timeout 5 --insecure -o'
+  option adb_fetchparm '-s --connect-timeout 10 --insecure -o'
 </code></pre>
   
 **receive adblock statistics via ubus:**
diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init
index 7c4a91c1c4..cf8a57b151 100755
--- a/net/adblock/files/adblock.init
+++ b/net/adblock/files/adblock.init
@@ -37,7 +37,6 @@ start_service()
     if [ $(/etc/init.d/adblock enabled; printf ${?}) -eq 0 ]
     then
         procd_open_instance "adblock"
-        procd_set_param env adb_procd="true"
         procd_set_param command "${adb_script}" "${@}"
         procd_set_param stdout 1
         procd_set_param stderr 1
@@ -52,7 +51,6 @@ reload_service()
 
 stop_service()
 {
-    export adb_procd="true"
     rc_procd "${adb_script}" stop
 }
 
@@ -73,7 +71,6 @@ resume()
 
 query()
 {
-    export adb_procd="true"
     rc_procd "${adb_script}" query "${1}"
 }
 
@@ -81,8 +78,6 @@ service_triggers()
 {
     local iface="$(uci -q get adblock.global.adb_iface)"
 
-    procd_open_trigger
-    procd_add_config_trigger "config.change" "adblock" /etc/init.d/adblock start
     if [ -z "${iface}" ]
     then
         procd_add_raw_trigger "interface.*.up" 1000 /etc/init.d/adblock start
@@ -92,5 +87,5 @@ service_triggers()
             procd_add_interface_trigger "interface.*.up" "${name}" /etc/init.d/adblock start
         done
     fi
-    procd_close_trigger
+    procd_add_config_trigger "config.change" "adblock" /etc/init.d/adblock start
 }
diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh
index c25541979c..a470d0bf7e 100755
--- a/net/adblock/files/adblock.sh
+++ b/net/adblock/files/adblock.sh
@@ -10,10 +10,11 @@
 #
 LC_ALL=C
 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
-adb_ver="2.3.1"
+adb_ver="2.4.0-2"
 adb_enabled=1
 adb_debug=0
 adb_backup=0
+adb_tldcomp=1
 adb_backupdir="/mnt"
 adb_whitelist="/etc/adblock/adblock.whitelist"
 adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}"
@@ -169,9 +170,12 @@ f_envcheck()
 #
 f_rmtemp()
 {
-    rm -f "${adb_tmpload}"
-    rm -f "${adb_tmpfile}"
-    rm -rf "${adb_tmpdir}"
+    if [ -d "${adb_tmpdir}" ]
+    then
+        rm -f "${adb_tmpload}"
+        rm -f "${adb_tmpfile}"
+        rm -rf "${adb_tmpdir}"
+    fi
 }
 
 # f_rmdns: remove dns related files & directories
@@ -321,7 +325,11 @@ f_log()
         then
             logger -t "adblock-[${adb_ver}] ${class}" "Please check the online documentation 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md'"
             f_rmtemp
-            f_rmdns
+            if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ]
+            then
+                f_rmdns
+                f_dnsrestart
+            fi
             exit 255
         fi
     fi
@@ -331,9 +339,10 @@ f_log()
 #
 f_main()
 {
-    local enabled url cnt sum_cnt=0
-    local src_name src_rset shalla_file shalla_archive list active_lists
+    local enabled url cnt sum_cnt=0 mem_total=0
+    local src_name src_rset shalla_archive list active_lists
     local sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')"
+    mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo" 2>/dev/null)"
 
     f_log "info " "start adblock processing ..."
     for src_name in ${adb_sources}
@@ -356,7 +365,7 @@ f_main()
 
         # download block list
         #
-        f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetch}"
+        f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetch}, memory: ${mem_total}"
         if [ "${src_name}" = "blacklist" ]
         then
             cat "${url}" 2>/dev/null > "${adb_tmpload}"
@@ -364,23 +373,19 @@ f_main()
         elif [ "${src_name}" = "shalla" ]
         then
             shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
-            shalla_file="${adb_tmpdir}/shallalist.txt"
             "${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}" 2>/dev/null
             adb_rc=${?}
             if [ ${adb_rc} -eq 0 ]
             then
-                > "${shalla_file}"
                 for category in ${adb_src_cat_shalla}
                 do
-                    tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}"
+                    tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${adb_tmpload}"
                     adb_rc=${?}
                     if [ ${adb_rc} -ne 0 ]
                     then
                         break
                     fi
                 done
-                cat "${shalla_file}" 2>/dev/null > "${adb_tmpload}"
-                rm -f "${shalla_file}"
             fi
             rm -f "${shalla_archive}"
             rm -rf "${adb_tmpdir}/BL"
@@ -389,13 +394,22 @@ f_main()
             adb_rc=${?}
         fi
 
-        # check download result and prepare domain output (incl. list backup/restore)
+        # check download result and prepare domain output (incl. tld compression, list backup & restore)
         #
         if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ]
         then
             awk "${src_rset}" "${adb_tmpload}" > "${adb_tmpfile}"
             if [ -s "${adb_tmpfile}" ]
             then
+                if [ ${adb_tldcomp} -eq 1 ]
+                then
+                    awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" | sort -u > "${adb_tmpload}"
+                    awk '{if(NR==1){tld=$NF};while(getline){if($NF !~ tld"\\."){print tld;tld=$NF}}print tld}' "${adb_tmpload}" > "${adb_tmpfile}"
+                    awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" > "${adb_tmpload}"
+                else
+                    sort -u "${adb_tmpfile}" > "${adb_tmpload}"
+                fi
+                mv -f "${adb_tmpload}" "${adb_tmpfile}"
                 f_list backup
             else
                 f_list restore
@@ -410,9 +424,9 @@ f_main()
         then
             if [ -s "${adb_tmpdir}/tmp.whitelist" ]
             then
-                grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}"
+                grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
             else
-                sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
+                cat "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
             fi
             adb_rc=${?}
             if [ ${adb_rc} -ne 0 ]
@@ -424,16 +438,19 @@ f_main()
         fi
     done
 
-    # sort/unique overall
+    # overall sort
     #
     for src_name in $(ls -dASr "${adb_tmpdir}/${adb_dnsprefix}"* 2>/dev/null)
     do
-        if [ -s "${adb_tmpdir}/blocklist.overall" ]
+        if [ ${mem_total} -ge 64000 ]
         then
-            sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/blocklist.overall" "${src_name}" | uniq -u > "${adb_tmpdir}/tmp.blocklist"
-            cat "${adb_tmpdir}/tmp.blocklist" > "${src_name}"
+            if [ -s "${adb_tmpdir}/blocklist.overall" ]
+            then
+                sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/blocklist.overall" "${src_name}" | uniq -u > "${adb_tmpdir}/tmp.blocklist"
+                mv -f "${adb_tmpdir}/tmp.blocklist" "${src_name}"
+            fi
+            cat "${src_name}" >> "${adb_tmpdir}/blocklist.overall"
         fi
-        cat "${src_name}" >> "${adb_tmpdir}/blocklist.overall"
         cnt="$(wc -l < "${src_name}")"
         sum_cnt=$((sum_cnt + cnt))
         list="${src_name/*./}"
@@ -449,6 +466,7 @@ f_main()
     #
     mv -f "${adb_tmpdir}/${adb_dnsprefix}"* "${adb_dnsdir}" 2>/dev/null
     chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null
+    f_rmtemp
     f_dnsrestart
     if [ "${adb_dnsup}" = "true" ]
     then
@@ -461,7 +479,6 @@ f_main()
             \"dns_backend\":\"${adb_dns}\",
             \"last_rundate\":\"$(/bin/date "+%d.%m.%Y %H:%M:%S")\",
             \"system\":\"${sysver}\"}}}}"
-        f_rmtemp
         return 0
     fi
     f_log "error" "dns backend restart with active block lists failed (${sysver})"
@@ -469,34 +486,31 @@ f_main()
 
 # handle different adblock actions
 #
-if [ "${adb_procd}" = "true" ]
-then
-    f_envload
-    case "${1}" in
-        stop)
-            f_rmtemp
-            f_rmdns
-            f_dnsrestart
-            ;;
-        restart)
-            f_rmtemp
-            f_rmdns
-            f_envcheck
-            f_main
-            ;;
-        suspend)
-            f_switch suspend
-            ;;
-        resume)
-            f_switch resume
-            ;;
-        query)
-            f_query "${2}"
-            ;;
-        *)
-            f_envcheck
-            f_main
-            ;;
-    esac
-fi
+f_envload
+case "${1}" in
+    stop)
+        f_rmtemp
+        f_rmdns
+        f_dnsrestart
+        ;;
+    restart)
+        f_rmtemp
+        f_rmdns
+        f_envcheck
+        f_main
+        ;;
+    suspend)
+        f_switch suspend
+        ;;
+    resume)
+        f_switch resume
+        ;;
+    query)
+        f_query "${2}"
+        ;;
+    *)
+        f_envcheck
+        f_main
+        ;;
+esac
 exit 0
-- 
2.30.2