From a721ef6f4875721334db38bdb386257714455c11 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Fri, 3 Apr 2020 20:33:13 +0200 Subject: [PATCH] adblock: release 4.0.2 * removed 'hphosts' from sources (discontinued) * fixed a "out of range" bug and another small issue in the f_dnsup function * add three new sources: 'anudeep', 'stopforumspam' and 'youtube' * changed 'list' behaviour, the source file has now a higher precedence than the archive file (see readme) * update readme, added missing parameters & more Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 4 +- net/adblock/files/README.md | 103 ++++++++++++++++-------------- net/adblock/files/adblock.init | 11 ++-- net/adblock/files/adblock.sh | 7 +- net/adblock/files/adblock.sources | 28 ++++++-- 5 files changed, 89 insertions(+), 64 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 445c22cd2e..a1f149b046 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -6,8 +6,8 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=4.0.1 -PKG_RELEASE:=2 +PKG_VERSION:=4.0.2 +PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 150f649511..aada8a0a12 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -11,6 +11,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but | adaway | x | S | mobile | [Link](https://github.com/AdAway/adaway.github.io) | | adguard | x | L | general | [Link](https://adguard.com) | | andryou | | L | compilation | [Link](https://gitlab.com/andryou/block/-/blob/master/readme.md) | +| anudeep | | M | compilation | [Link](https://github.com/anudeepND/blacklist) | | bitcoin | | S | mining | [Link](https://github.com/hoshsadiq/adblock-nocoin-list) | | disconnect | x | S | general | [Link](https://disconnect.me) | | dshield | | XL | general | [Link](https://dshield.org) | @@ -18,7 +19,6 @@ A lot of people already use adblocker plugins within their desktop browsers, but | energized_blu | | XL | compilation | [Link](https://github.com/EnergizedProtection/block) | | energized_porn | | XXL | compilation+porn | [Link](https://github.com/EnergizedProtection/block) | | energized_unified | | XXL | compilation | [Link](https://github.com/EnergizedProtection/block) | -| hphosts | | M | general | [Link](https://hosts-file.net) | | malwaredomains | | M | malware | [Link](https://malwaredomains.com) | | malwarelist | | S | malware | [Link](https://www.malwaredomainlist.com) | | notracking | | XL | tracking | [Link](https://github.com/notracking/hosts-blocklists) | @@ -39,17 +39,19 @@ A lot of people already use adblocker plugins within their desktop browsers, but | smarttv | | S | smarttv | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | spam404 | | S | general | [Link](https://github.com/Dawsey21) | | stevenblack | | L | compilation | [Link](https://github.com/StevenBlack/hosts) | +| stopforumspam | | S | spam | [Link](https://www.stopforumspam.com) | | sysctl | | M | general | [Link](http://sysctl.org/cameleon) | | utcapitole | | L | general | [Link](https://dsi.ut-capitole.fr/blacklists/index_en.php) | | utcapitole_porn | | XXL | general+porn | [Link](https://dsi.ut-capitole.fr/blacklists/index_en.php) | | whocares | | M | general | [Link](https://someonewhocares.org) | | winhelp | | S | general | [Link](http://winhelp2002.mvps.org) | | winspy | | S | win_telemetry | [Link](https://github.com/crazy-max/WindowsSpyBlocker) | +| youtube | | M | youtube | [Link](https://github.com/kboghdady/youTube_ads_4_pi-hole) | | yoyo | x | S | general | [Link](http://pgl.yoyo.org/adservers) | * List of supported and fully pre-configured adblock sources, already active sources are pre-selected. To avoid OOM errors, please do not select too many lists! - List size recommendations as follows: + List size information with the respective domain ranges as follows: • S (-10k), M (10k-30k) and L (30k-80k) should work for 128 MByte devices, • XL (80k-200k) should work for 256-512 MByte devices, • XXL (200k-) needs more RAM and Multicore support, e.g. x86 or raspberry devices. @@ -91,6 +93,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * A download utility with SSL support: 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries, 'aria2c' or 'curl' is required * Optional E-Mail notification support: for E-Mail notifications you need to install the additional 'msmtp' package * Optional DNS Query Report support: for DNS reporting you need to install the additional package 'tcpdump-mini' or 'tcpdump' +* Optional support for gnu awk as alternative to the busybox default, install the additional package 'gawk' ## Installation & Usage * Update your local opkg repository (_opkg update_) @@ -101,7 +104,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but ## Adblock CLI Options * All important adblock functions are accessible via CLI as well. -Please note: The 'status' command in 19.07.x and TurrisOS is only available via 'status_service' +Please note: The 'status' command in 19.07.x and TurrisOS is only available via 'status_service' 

 /etc/init.d/adblock 
 Syntax: /etc/init.d/adblock [command]
@@ -126,46 +129,48 @@ Available commands:
 ## Adblock Config Options
 * Usually the auto pre-configured adblock setup works quite well and no manual overrides are needed
 
-| Option            | Default                   | Description/Valid Values                                                                       |
-| :---------------- | :------------------------ | :--------------------------------------------------------------------------------------------- |
-| adb_enabled       | 1, enabled                | set to 0 to disable the adblock service                                                        |
-| adb_dns           | -, auto-detected          | 'dnsmasq', 'unbound', 'named', 'kresd' or 'raw'                                                |
-| adb_fetchutil     | -, auto-detected          | 'uclient-fetch', 'wget', 'curl' or 'aria2c'                                                    |
-| adb_fetchparm     | -, auto-detected          | special config options for the selected download utility                                       |
-| adb_trigger       | -, not set                | trigger network interface or 'not set' to use a time-based startup                             |
-| adb_triggerdelay  | 2                         | additional trigger delay in seconds before adblock processing begins                           |
-| adb_debug         | 0, disabled               | set to 1 to enable the debug output                                                            |
-| adb_nice          | 0, standard prio.         | valid nice level range 0-19 of the adblock processes                                           |
-| adb_forcedns      | 0, disabled               | set to 1 to force DNS requests to the local resolver                                           |
-| adb_maxqueue      | 4                         | size of the download queue to handle downloads & list processing in parallel                   |
-| adb_dnsdir        | -, auto-detected          | path for the generated blocklist file 'adb_list.overall'                                       |
-| adb_dnstimeout    | 10                        | timeout in seconds to wait for a successful DNS backend restart                                |
-| adb_dnsinstance   | 0, first instance         | set to the relevant dns backend instance used by adblock (dnsmasq only)                        |
-| adb_dnsfilereset  | 0, disabled               | set to 1 to purge the final DNS blocklist file after DNS backend loading                       |
-| adb_dnsflush      | 0, disabled               | set to 1 to flush the DNS Cache before & after adblock processing                              |
-| adb_dnsinotify    | -, not set                | set to 1 to prevent adblock triggered restarts for DNS backends with autoload functions        |
-| adb_dnsallow      | -, not set                | set to 1 to disable selective DNS whitelisting (RPZ pass through)                              |
-| adb_lookupdomain  | example.com               | external domain to check for a successful DNS backend restart or 'false' to disable this check |
-| adb_portlist      | 53 853 5353               | space separated list of firewall ports which should be redirected locally                      |
-| adb_report        | 0, disabled               | set to 1 to enable the background tcpdump gathering process for reporting                      |
-| adb_reportdir     | /tmp                      | path for DNS related report files                                                              |
-| adb_repiface      | -, auto-detected          | name of the reporting interface or 'any' used by tcpdump                                       |
-| adb_replisten     | 53                        | space separated list of reporting port(s) used by tcpdump                                      |
-| adb_repchunkcnt   | 5                         | report chunk count used by tcpdump                                                             |
-| adb_repchunksize  | 1                         | report chunk size used by tcpdump in MB                                                        |
-| adb_backup        | 1, enabled                | set to 0 to disable the backup function                                                        |
-| adb_backupdir     | /tmp                      | path for adblock backups                                                                       |
-| adb_tmpbase       | /tmp                      | path for all adblock related runtime operations, e.g. downloading, sorting, merging etc.       |
-| adb_safesearch    | 0, disabled               | set to 1 to enforce SafeSearch for google, bing, duckduckgo, yandex, youtube and pixabay       |
-| adb_safesearchmod | 0, disabled               | set to 1 to enable moderate SafeSearch filters for youtube                                     |
-| adb_mail          | 0, disabled               | set to 1 to enable notification E-Mails in case of a processing errors                         |
-| adb_mailreceiver  | -, not set                | receiver address for adblock notification E-Mails                                              |
-| adb_mailsender    | no-reply@adblock          | sender address for adblock notification E-Mails                                                |
-| adb_mailtopic     | adblock notification | topic for adblock notification E-Mails                                                         |
-| adb_mailprofile   | adb_notify                | mail profile used in 'msmtp' for adblock notification E-Mails                                  |
-| adb_mailcnt       | 0                         | minimum domain count to trigger E-Mail notifications                                           |
-| adb_jail          | 0                         | set to 1 to enable the additional, restrictive 'adb_list.jail' creation                        |
-| adb_jaildir       | /tmp                      | path for the generated jail list                                                               |
+| Option            | Default                            | Description/Valid Values                                                                       |
+| :---------------- | :--------------------------------- | :--------------------------------------------------------------------------------------------- |
+| adb_enabled       | 1, enabled                         | set to 0 to disable the adblock service                                                        |
+| adb_srcarc        | -, /etc/adblock/adblock.sources.gz | full path to the used adblock source archive                                                   |
+| adb_srcfile       | -, /tmp/adb_sources.json           | full path to the used adblock source file, which has a higher precedence than the archive file |
+| adb_dns           | -, auto-detected                   | 'dnsmasq', 'unbound', 'named', 'kresd' or 'raw'                                                |
+| adb_fetchutil     | -, auto-detected                   | 'uclient-fetch', 'wget', 'curl' or 'aria2c'                                                    |
+| adb_fetchparm     | -, auto-detected                   | config options for the selected download utility                                               |
+| adb_trigger       | -, not set                         | trigger network interface or 'not set' to use a time-based startup                             |
+| adb_triggerdelay  | 2                                  | additional trigger delay in seconds before adblock processing begins                           |
+| adb_debug         | 0, disabled                        | set to 1 to enable the debug output                                                            |
+| adb_nice          | 0, standard prio.                  | valid nice level range 0-19 of the adblock processes                                           |
+| adb_forcedns      | 0, disabled                        | set to 1 to force DNS requests to the local resolver                                           |
+| adb_maxqueue      | 4                                  | size of the download queue to handle downloads & list processing in parallel                   |
+| adb_dnsdir        | -, auto-detected                   | path for the generated blocklist file 'adb_list.overall'                                       |
+| adb_dnstimeout    | 10                                 | timeout in seconds to wait for a successful DNS backend restart                                |
+| adb_dnsinstance   | 0, first instance                  | set to the relevant dns backend instance used by adblock (dnsmasq only)                        |
+| adb_dnsfilereset  | 0, disabled                        | set to 1 to purge the final DNS blocklist file after DNS backend loading                       |
+| adb_dnsflush      | 0, disabled                        | set to 1 to flush the DNS Cache before & after adblock processing                              |
+| adb_dnsinotify    | -, not set                         | set to 1 to prevent adblock triggered restarts for DNS backends with autoload functions        |
+| adb_dnsallow      | -, not set                         | set to 1 to disable selective DNS whitelisting (RPZ pass through)                              |
+| adb_lookupdomain  | example.com                        | external domain to check for a successful DNS backend restart or 'false' to disable this check |
+| adb_portlist      | 53 853 5353                        | space separated list of firewall ports which should be redirected locally                      |
+| adb_report        | 0, disabled                        | set to 1 to enable the background tcpdump gathering process for reporting                      |
+| adb_reportdir     | /tmp                               | path for DNS related report files                                                              |
+| adb_repiface      | -, auto-detected                   | name of the reporting interface or 'any' used by tcpdump                                       |
+| adb_replisten     | 53                                 | space separated list of reporting port(s) used by tcpdump                                      |
+| adb_repchunkcnt   | 5                                  | report chunk count used by tcpdump                                                             |
+| adb_repchunksize  | 1                                  | report chunk size used by tcpdump in MB                                                        |
+| adb_backup        | 1, enabled                         | set to 0 to disable the backup function                                                        |
+| adb_backupdir     | /tmp                               | path for adblock backups                                                                       |
+| adb_tmpbase       | /tmp                               | path for all adblock related runtime operations, e.g. downloading, sorting, merging etc.       |
+| adb_safesearch    | 0, disabled                        | set to 1 to enforce SafeSearch for google, bing, duckduckgo, yandex, youtube and pixabay       |
+| adb_safesearchmod | 0, disabled                        | set to 1 to enable moderate SafeSearch filters for youtube                                     |
+| adb_mail          | 0, disabled                        | set to 1 to enable notification E-Mails in case of a processing errors                         |
+| adb_mailreceiver  | -, not set                         | receiver address for adblock notification E-Mails                                              |
+| adb_mailsender    | no-reply@adblock                   | sender address for adblock notification E-Mails                                                |
+| adb_mailtopic     | adblock notification          | topic for adblock notification E-Mails                                                         |
+| adb_mailprofile   | adb_notify                         | mail profile used in 'msmtp' for adblock notification E-Mails                                  |
+| adb_mailcnt       | 0                                  | minimum domain count to trigger E-Mail notifications                                           |
+| adb_jail          | 0                                  | set to 1 to enable the additional, restrictive 'adb_list.jail' creation                        |
+| adb_jaildir       | /tmp                               | path for the generated jail list                                                               |
 
 ## Examples
 **Change the DNS backend to 'unbound':**  
@@ -190,7 +195,7 @@ and at the end of the file add:
 
 **Change the DNS backend to 'kresd':**  
 Adblock deposits the final blocklist 'adb_list.overall' in '/etc/kresd', no further configuration needed.  
-Please note: The knot-resolver (kresd) is only available on Turris devices and does not support the SafeSearch functionality yet.
+Please note: The knot-resolver (kresd) is only available on Turris devices and does not support the SafeSearch functionality yet.
 
 **Enable E-Mail notification via 'msmtp':**  
 To use the email notification you have to install & configure the package 'msmtp'.  
@@ -220,7 +225,7 @@ To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/in
 /etc/init.d/adblock status_service
 ::: adblock runtime information
   + adblock_status  : enabled
-  + adblock_version : 4.0.0
+  + adblock_version : 4.0.2
   + blocked_domains : 52420
   + active_sources  : adaway adguard andryou bitcoin disconnect winspy yoyo 
   + dns_backend     : kresd, /etc/kresd
@@ -228,10 +233,10 @@ To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/in
   + run_ifaces      : trigger: trm_wwan, report: br-lan
   + run_directories : base: /tmp, backup: /tmp, report: /tmp, jail: /tmp
   + run_flags       : backup: 1, reset: 0, flush: 0, force: 1, search: 0, report: 1, mail: 0, jail: 0
-  + last_run        : start, 0m 17s, 496/198/218, 27.03.2020 08:55:14
+  + last_run        : start, 0m 17s, 496/198/218, 03.04.2020 08:55:14
   + system          : CZ.NIC Turris Mox Board, TurrisOS 5.1.0 81264ebb51991aa2d17489852854e3b5ec3f514d
-The 'last\_run' line includes the used start type, the duration of the last run, the memory footprint after DNS backend loading (total/free/available) and the date/time of the last run. +The 'last\_run' line includes the used start type, the run duration, the memory footprint after DNS backend loading (total/free/available) and the date/time of the last run. **Edit, add new adblock sources:** The adblock blocklist sources are stored in an external, compressed JSON file '/etc/adblock/adblock.sources.gz'. @@ -267,7 +272,9 @@ A valid JSON source object contains the following required information, e.g.: }, [...] -Add an unique object name, make the required changes to 'url', 'rule', 'size' and 'descurl' and finally compress the changed JSON file _gzip /etc/adblock/adblock.sources.gz_ to use the new source object in adblock. +Add an unique object name, make the required changes to 'url', 'rule', 'size' and 'descurl' and finally compress the changed JSON file _gzip /etc/adblock/adblock.sources.gz_ to use the new source object in adblock. +Please note: if you're going to add new sources on your own, please make a copy of the default file and work with that copy further on, cause the default will be overwritten with every adblock update. To reference your copy set the option 'adb\_srcarc' which points by default to '/etc/adblock/adblock.sources.gz' +Please note: when adblock starts, it looks for the uncompressed 'adb\_srcfile', only if this file is not found the archive 'adb\_srcarc' is unpacked once and then the uncompressed file is used ## Support Please join the adblock discussion in this [forum thread](https://forum.openwrt.org/t/adblock-support-thread/507) or contact me by mail diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init index fc084ec141..c3fd46c83b 100755 --- a/net/adblock/files/adblock.init +++ b/net/adblock/files/adblock.init @@ -119,11 +119,14 @@ list() src_archive="$(uci_get adblock global adb_srcarc "/etc/adblock/adblock.sources.gz")" src_file="$(uci_get adblock global adb_srcfile "/tmp/adb_sources.json")" src_enabled="$(uci -q show adblock.global.adb_sources)" - if [ -r "${src_archive}" ] + if [ ! -r "${src_file}" ] then - zcat "${src_archive}" > "${src_file}" - else - printf "%s\\n" "::: adblock source archive '${src_archive}' not found" + if [ -r "${src_archive}" ] + then + zcat "${src_archive}" > "${src_file}" + else + printf "%s\\n" "::: adblock source archive '${src_archive}' not found" + fi fi if [ -r "${src_file}" ] then diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index 95d7c80b54..ef19f85452 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -11,7 +11,7 @@ export LC_ALL=C export PATH="/usr/sbin:/usr/bin:/sbin:/bin" set -o pipefail -adb_ver="4.0.1" +adb_ver="4.0.2" adb_enabled=0 adb_debug=0 adb_forcedns=0 @@ -39,9 +39,9 @@ adb_blacklist="/etc/adblock/adblock.blacklist" adb_whitelist="/etc/adblock/adblock.whitelist" adb_ubusservice="/etc/adblock/adblock.monitor" adb_mailservice="/etc/adblock/adblock.mail" -adb_srcarc="/etc/adblock/adblock.sources.gz" adb_dnsfile="${adb_dnsprefix}.overall" adb_dnsjail="${adb_dnsprefix}.jail" +adb_srcarc="/etc/adblock/adblock.sources.gz" adb_srcfile="${adb_tmpbase}/adb_sources.json" adb_rtfile="${adb_tmpbase}/adb_runtime.json" adb_loggercmd="$(command -v logger)" @@ -635,7 +635,7 @@ f_dnsup() "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 restart_rc="${?}" fi - if [ "${restart_rc}" -eq 0 ] + if [ "${restart_rc}" = "0" ] then while [ "${cnt}" -le "${adb_dnstimeout}" ] do @@ -655,6 +655,7 @@ f_dnsup() fi else sleep ${adb_dnstimeout} + cnt=${adb_dnstimeout} out_rc=0 break fi diff --git a/net/adblock/files/adblock.sources b/net/adblock/files/adblock.sources index 806fab094a..c01263873e 100644 --- a/net/adblock/files/adblock.sources +++ b/net/adblock/files/adblock.sources @@ -20,6 +20,13 @@ "focus": "compilation", "descurl": "https://gitlab.com/andryou/block/-/blob/master/readme.md" }, + "anudeep": { + "url": "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt", + "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", + "size": "M", + "focus": "compilation", + "descurl": "https://github.com/anudeepND/blacklist" + }, "bitcoin": { "url": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt", "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", @@ -69,13 +76,6 @@ "focus": "compilation", "descurl": "https://github.com/EnergizedProtection/block" }, - "hphosts": { - "url": "https://hosts-file.net/ad_servers.txt", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "general", - "descurl": "https://hosts-file.net" - }, "malwaredomains": { "url": "http://mirror.espoch.edu.ec/malwaredomains/justdomains", "rule": "/^([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", @@ -222,6 +222,13 @@ "focus": "compilation", "descurl": "https://github.com/StevenBlack/hosts" }, + "stopforumspam": { + "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole.txt", + "rule": "/^[[:space:]]*([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", + "size": "S", + "focus": "spam", + "descurl": "https://www.stopforumspam.com" + }, "sysctl": { "url": "http://sysctl.org/cameleon/hosts", "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", @@ -270,6 +277,13 @@ "focus": "win_telemetry", "descurl": "https://github.com/crazy-max/WindowsSpyBlocker" }, + "youtube": { + "url": "https://raw.githubusercontent.com/kboghdady/youTube_ads_4_pi-hole/master/black.list", + "rule": "/^([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", + "size": "M", + "focus": "youtube", + "descurl": "https://github.com/kboghdady/youTube_ads_4_pi-hole" + }, "yoyo": { "url": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext", "rule": "/^([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", -- 2.30.2