From 262f4551e271111cf1365e06e9302388f19b28c5 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Mon, 2 May 2016 19:43:46 +0200 Subject: [PATCH] adblock: update to 1.1.1 * discard uncommitted uci changes in case of processing errors * init script (stop action) now removes all 'adb_src_count' & 'adb_src_timestamp' entries in config * updated documentation Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 2 +- net/adblock/files/README.md | 36 +++++++++++++++++++---------- net/adblock/files/adblock-helper.sh | 21 ++++++++++------- net/adblock/files/adblock-update.sh | 18 +++++++-------- net/adblock/files/adblock.init | 19 +++++++++++---- 5 files changed, 62 insertions(+), 34 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 5597daea69..f4fc48a2cf 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=1.1.0 +PKG_VERSION:=1.1.1 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 7e0c171b5f..1cbf8ced10 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -147,21 +147,33 @@ This rule removes _all_ domains from the blocklists with this string in it, i.e. photos.daily-deals.analoganalytics.com adblockanalytics.com -**example to add [rolist+easylist](https://easylist-downloads.adblockplus.org/rolist+easylist.txt) as a new source:** +**example to add a new blocklist sources:** 

-add a new source section in adblock config, take an existing easylist source as a template and change only the source name, the url and the description
-
-config source 'ruadlist'
-        option enabled '0'
-        option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt'
-        option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}'
-        option adb_src_desc 'focus on russian ad related domains plus generic easylist additions, weekly updates, approx. 2.000 entries'
+1. the easy way ...
+example: https://easylist-downloads.adblockplus.org/rolist+easylist.txt
+adblock already supports an easylist source, called 'ruadlist'. To add the additional local easylist
+as a new source, copy the existing config source 'ruadlist' section and change only 
+the source name, the url and the description - that's all!
 
 config source 'rolist'
-        option enabled '0'
-        option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt'
-        option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}'
-        option adb_src_desc 'focus on romanian ad related domains plus generic easylist additions, weekly updates, approx. 600 entries'
+  option enabled '0'
+  option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt'
+  option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}'
+  option adb_src_desc 'focus on romanian ad related domains plus generic easylist additions, weekly updates, approx. 600 entries'
+
+2. a bit harder ...
+to add a really new source with different domain/host format you have to write a suitable
+awk one-liner on your own, so basic awk skills are needed. As a starting point check the already
+existing awk strings (adb_src_rset) in adblock config, maybe you need only small changes for your individual list.
+Download the desired list and test your new awk string locally with:
+  cat new.list | awk 'fs__individual search__search core__result'
+  'fs' => field separator (optional)
+  'individual search' => individual search part to filter out needless list information
+  'search core' => always '([A-Za-z0-9_-]+\.){1,}[A-Za-z]+', this is part of all list sources and should be unchanged
+  'result' => always '{print tolower(\$n)}', only the output column 'n' may vary
+the output result should be a sequential list with one domain/host per line - nothing more.
+
+If your awk one-liner works quite well, add a new source section in adblock config and test your new source
## Background diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh index c99f58d56e..b729fc4e3c 100644 --- a/net/adblock/files/adblock-helper.sh +++ b/net/adblock/files/adblock-helper.sh @@ -110,7 +110,7 @@ f_envload() if [ -z "${adb_enabled}" ] || [ -z "${adb_cfgversion}" ] || [ "${adb_cfgversion}" != "${adb_scriptver%.*}" ] then rc=125 - f_log "no valid adblock configuration found, please merge latest changes from '/etc/config/adblock.opkg' manually" "${rc}" + f_log "outdated adblock configuration found, please use latest version from '/etc/adblock/adblock.conf.default'" "${rc}" f_exit fi if [ $((adb_enabled)) -ne 1 ] @@ -128,6 +128,7 @@ f_envload() adb_dnsdir="/tmp/dnsmasq.d" adb_dnsprefix="adb_list" adb_fetch="/usr/bin/wget" + adb_uci="/sbin/uci" unset adb_srclist adb_revsrclist adb_errsrclist # get lan ip addresses @@ -476,10 +477,10 @@ f_cntconfig() then count=$((count / 2)) fi - uci_set "adblock" "${src_name}" "adb_src_count" "${count}" + ${adb_uci} -q set "adblock.${src_name}.adb_src_count=${count}" count_sum=$((count_sum + count)) done - uci_set "adblock" "global" "adb_overall_count" "${count_sum}" + ${adb_uci} -q set "adblock.global.adb_overall_count=${count_sum}" } # f_rmconfig: remove counters & timestamps in given config sections @@ -493,10 +494,10 @@ f_rmconfig() src_name="${list/*./}" if [ -n "${restore_done}" ] then - uci_set "adblock" "${src_name}" "adb_src_timestamp" "list restored" + ${adb_uci} -q set "adblock.${src_name}.adb_src_timestamp=list restored" else - uci_remove "adblock" "${src_name}" "adb_src_count" - uci_remove "adblock" "${src_name}" "adb_src_timestamp" + ${adb_uci} -q delete "adblock.${src_name}.adb_src_count" + ${adb_uci} -q delete "adblock.${src_name}.adb_src_timestamp" fi done unset restore_done @@ -598,14 +599,18 @@ f_exit() ipv6_adblock="$(${iptv6} -t nat -vnL | awk '$10 ~ /^adb-nat$/ {sum += $1} END {printf sum}')" ipv6_adblock="$((${ipv6_adblock} + $(${iptv6} -vnL | awk '$10 ~ /^adb-(fwd|out)$/ {sum += $1} END {printf sum}')))" fi - if [ -n "$(uci changes adblock)" ] + if [ -n "$(${adb_uci} -q changes adblock)" ] then - uci_commit "adblock" + ${adb_uci} -q commit "adblock" fi f_log "firewall statistics (IPv4/IPv6): ${ipv4_adblock}/${ipv6_adblock} ad related packets blocked" f_log "domain adblock processing finished successfully (${adb_scriptver}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))" elif [ $((rc)) -gt 0 ] then + if [ -n "$(${adb_uci} -q changes adblock)" ] + then + ${adb_uci} -q revert "adblock" + fi f_log "domain adblock processing failed (${adb_scriptver}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))" else rc=0 diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh index 54581b5f6d..fb031dd928 100755 --- a/net/adblock/files/adblock-update.sh +++ b/net/adblock/files/adblock-update.sh @@ -36,7 +36,7 @@ fi # get current directory, script- and openwrt version # adb_scriptdir="${0%/*}" -adb_scriptver="1.1.0" +adb_scriptver="1.1.1" openwrt_version="$(cat /etc/openwrt_version)" # source in adblock function library @@ -75,14 +75,14 @@ do eval "url=\"\${adb_src_${src_name}}\"" eval "src_rset=\"\${adb_src_rset_${src_name}}\"" adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}" - list_time="$(uci_get "adblock" "${src_name}" "adb_src_timestamp")" + list_time="$(${adb_uci} -q get "adblock.${src_name}.adb_src_timestamp")" f_log "=> processing adblock source '${src_name}'" # check 'url' and 'src_rset' values # if [ -z "${url}" ] || [ -z "${src_rset}" ] then - uci_set "adblock" "${src_name}" "adb_src_timestamp" "broken config" + ${adb_uci} -q set "adblock.${src_name}.adb_src_timestamp=broken config" f_log " broken source configuration, check 'adb_src' and 'adb_src_rset' in config" continue fi @@ -162,7 +162,7 @@ do unset tmp_domains elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ] then - uci_set "adblock" "${src_name}" "adb_src_timestamp" "empty download" + ${adb_uci} -q set "adblock.${src_name}.adb_src_timestamp=empty download" f_log " empty source download finished" continue else @@ -173,7 +173,7 @@ do else adb_errsrclist="${adb_errsrclist} -o -name ${adb_dnsprefix}.${src_name}" fi - uci_set "adblock" "${src_name}" "adb_src_timestamp" "download failed" + ${adb_uci} -q set "adblock.${src_name}.adb_src_timestamp=download failed" f_log " source download failed" continue fi @@ -205,14 +205,14 @@ do # if [ $((rc)) -eq 0 ] then - uci_set "adblock" "${src_name}" "adb_src_timestamp" "${url_time}" + ${adb_uci} -q set "adblock.${src_name}.adb_src_timestamp=${url_time}" f_log " domain merging finished" else f_log " domain merging failed" "${rc}" f_restore fi else - uci_set "adblock" "${src_name}" "adb_src_timestamp" "empty domain input" + ${adb_uci} -q set "adblock.${src_name}.adb_src_timestamp=empty domain input" f_log " empty domain input received" continue fi @@ -239,7 +239,7 @@ fi if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ] then f_rmconfig "${rm_done}" - f_log "disabled adblock lists removed" + f_log "remove disabled adblock lists" elif [ $((rc)) -ne 0 ] && [ -n "${rm_done}" ] then f_log "error during removal of disabled adblock lists" "${rc}" @@ -295,7 +295,7 @@ fi # restart & check dnsmasq with newly generated set of adblock lists # f_cntconfig -adb_count="$(uci_get "adblock" "global" "adb_overall_count")" +adb_count="$(${adb_uci} -q get "adblock.global.adb_overall_count")" if [ -n "${adb_revsrclist}" ] || [ -n "${rm_done}" ] || [ -n "${restore_done}" ] then /etc/init.d/dnsmasq restart diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init index f8b1262997..7543553339 100755 --- a/net/adblock/files/adblock.init +++ b/net/adblock/files/adblock.init @@ -8,6 +8,7 @@ adb_pid="${$}" adb_script="/usr/bin/adblock-update.sh" adb_pidfile="/var/run/adblock.pid" adb_logger="/usr/bin/logger" +adb_uci="/sbin/uci" if [ -t 1 ] then @@ -24,6 +25,15 @@ then exit 255 fi +remove_config() +{ + local value opt section="${1}" options="adb_src_timestamp adb_src_count" + for opt in ${options} + do + ${adb_uci} -q delete "adblock.${section}.${opt}" + done +} + boot() { return 0 @@ -53,10 +63,11 @@ stop() rc=${?} if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ] then - . "/usr/bin/adblock-helper.sh" - uci_remove "adblock" "global" "adb_overall_count" - f_rmconfig "${rm_done}" - uci_commit "adblock" + . "/lib/functions.sh" + config_load adblock + config_foreach remove_config source + ${adb_uci} -q delete "adblock.global.adb_overall_count" + ${adb_uci} -q commit "adblock" /etc/init.d/dnsmasq restart /etc/init.d/firewall restart fi -- 2.30.2