simple-adblock: bugfix: proper error reporting on failed downloads; lists update... 12334/head
authorStan Grishin <stangri@melmac.net>
Wed, 27 May 2020 00:26:13 +0000 (00:26 +0000)
committerStan Grishin <stangri@melmac.net>
Wed, 27 May 2020 00:26:13 +0000 (00:26 +0000)
Signed-off-by: Stan Grishin <stangri@melmac.net>
net/simple-adblock/Makefile
net/simple-adblock/files/README.md
net/simple-adblock/files/simple-adblock.conf
net/simple-adblock/files/simple-adblock.conf.update [new file with mode: 0644]
net/simple-adblock/files/simple-adblock.init

index 793cb14d5a6909eda846cbeed19d4b7d52b74315..2c45fe9ee20d5e3cf935641f710d53b2032f77b7 100644 (file)
@@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=simple-adblock
 PKG_VERSION:=1.8.3
-PKG_RELEASE:=3
+PKG_RELEASE:=9
 PKG_MAINTAINER:=Stan Grishin <stangri@melmac.net>
 PKG_LICENSE:=GPL-3.0-or-later
 
@@ -21,7 +21,7 @@ endef
 
 define Package/simple-adblock/description
 This service provides DNSMASQ or Unbound based ad blocking.
-Please see the project's README at github for further information.
+Please see the project's README on GitHub for further information.
 
 endef
 
@@ -46,6 +46,8 @@ define Package/simple-adblock/install
        $(INSTALL_BIN) $(PKG_BUILD_DIR)/files/simple-adblock.init $(1)/etc/init.d/simple-adblock
        $(INSTALL_DIR) $(1)/etc/config
        $(INSTALL_CONF) ./files/simple-adblock.conf $(1)/etc/config/simple-adblock
+       $(INSTALL_DIR) $(1)/tmp
+       $(INSTALL_CONF) ./files/simple-adblock.conf.update $(1)/tmp/simple-adblock.conf.update
        $(INSTALL_DIR) $(1)/etc/hotplug.d/iface
        $(INSTALL_BIN) ./files/simple-adblock.hotplug $(1)/etc/hotplug.d/iface/80-simple-adblock
 endef
@@ -54,6 +56,7 @@ define Package/simple-adblock/postinst
        #!/bin/sh
        # check if we are on real system
        if [ -z "$${IPKG_INSTROOT}" ]; then
+               sed -f /tmp/simple-adblock.conf.update -i /etc/config/simple-adblock || true
                /etc/init.d/simple-adblock enable
        fi
        exit 0
index 8582f9e16422ba27e278775b8e13d5d8a8ff81ee..f5fd095a847f9a0a4785258b9eb4aa0c9443d390 100644 (file)
@@ -19,63 +19,70 @@ A simple DNSMASQ/Unbound-based AdBlocking service for OpenWrt/LEDE Project.
 - Blocks ads inside browsers with [DNS-over-HTTPS proxy](https://en.wikipedia.org/wiki/DNS_over_HTTPS) built-in, like [Mozilla Firefox](https://support.mozilla.org/en-US/kb/firefox-dns-over-https#w_about-dns-over-https) or [Google Chrome/Chromium](https://blog.chromium.org/2019/09/experimenting-with-same-provider-dns.html) -- with the ```dnsmasq.ipset``` option.
 - Proudly made in Canada, using locally-sourced electrons.
 
-If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out [net/adblock](https://github.com/openwrt/packages/tree/master/net/adblock/files).
+If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out ```net/adblock``` on [GitHub](https://github.com/openwrt/packages/tree/master/net/adblock/files)/[jsDelivr](https://cdn.jsdelivr.net/gh/openwrt/packages@master/net/adblock/files/README.md).
 
 ## Screenshots (luci-app-simple-adblock)
 
 Service Status
 
-![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-status.png "Service Status")
+![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-status.png "Service Status")
 
 Configuration - Basic Configuration
 
-![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-config-basic.png "Configuration - Basic Configuration")
+![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-config-basic.png "Configuration - Basic Configuration")
 
 Configuration - Advanced Configuration
 
-![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-config-advanced.png "Configuration - Advanced Configuration")
+![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-config-advanced.png "Configuration - Advanced Configuration")
 
 Whitelist and Blocklist Management
 
-![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-lists.png "Whitelist and Blocklist Management")
+![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-lists.png "Whitelist and Blocklist Management")
 
 ## Requirements
 
-This service requires the following packages to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound``` and either ```ca-certificates```, ```wget``` and ```libopenssl``` (for OpenWrt 15.05.1) or ```uclient-fetch``` and ```libustream-mbedtls``` (for LEDE Project and OpenWrt 18.06.xx or newer). Additionally installation of ```coreutils-sort``` is highly recommended as it speeds up blocklist processing.
+To download the hosts/domains files this service requires one of the following packages to be installed on your device: ```wget``` or ```uclient-fetch``` or ```curl```. If you want to download lists over HTTPS (recommended), you also need to install support for HTTPS/SSL downloads on your device. For most devices starting with OpenWrt 15.05.1 and up the following commands will install the minimal requirements:
 
-To satisfy the requirements for connect to your router via ssh and run the following commands:
+```sh
+opkg update; if ubus -S call system board | grep -q '15.05'; then
+opkg install ca-certificates wget libopenssl; else
+opkg install uclient-fetch libustream-mbedtls ca-bundle ca-certificates; fi
+```
 
-### How to use DNSMASQ ipset
+### Requirements for file:// Scheme
 
-The ```dnsmasq.ipset``` option requires you to install ```ipset``` and  ```dnsmasq-full``` instead of the ```dnsmasq```. To do that, connect to your router via ssh and run the following command:
+If you want to include some local files as the blocklists or whitelists, you need to install ```curl```:
 
 ```sh
-opkg update; opkg remove dnsmasq; opkg install dnsmasq-full ipset;
+opkg update; opkg install curl;
 ```
 
-### OpenWrt 15.05.1 Requirements
+### Requirements for DNS Resolver
 
-```sh
-opkg update; opkg install ca-certificates wget libopenssl coreutils-sort dnsmasq
-```
+In order to actually block the ads, this service requires one of the following DNS resolvers to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound```. The ```dnsmasq``` package is usually included in the default OpenWrt installation.
 
-### LEDE Project 17.01.x and OpenWrt 18.xx (or newer) Requirements
+If you want to use the [```dnsmasq.ipset``` option](#dns-resolution-option) you need to install ```ipset``` and  ```dnsmasq-full``` instead of the ```dnsmasq```. To do that, connect to your router via ssh and run the following command:
 
 ```sh
-opkg update; opkg install uclient-fetch libustream-mbedtls coreutils-sort dnsmasq
+opkg update; cd /tmp/; opkg download dnsmasq-full; opkg install ipset;
+opkg remove dnsmasq; opkg install /tmp/dnsmasq-full*; rm -f /tmp/dnsmasq-full*;
 ```
 
-### IPv6 Support
+### Requirements for IPv6 Support
 
 For IPv6 support additionally install ```ip6tables-mod-nat``` and ```kmod-ipt-nat6``` packages from Web UI or run the following in the command line:
 
 ```sh
-opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6
+opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6;
 ```
 
-### Speed Up Blocklist Processing
+### Requirements for Faster Blocklist Processing
 
-The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run ```opkg --force-overwrite install coreutils-sort```.
+The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run the following command: 
+
+```sh
+opkg --force-overwrite install coreutils-sort
+```
 
 ## Unmet Dependencies
 
@@ -89,11 +96,11 @@ Install ```simple-adblock``` and ```luci-app-simple-adblock``` packages from Web
 opkg update; opkg install simple-adblock luci-app-simple-adblock
 ```
 
-If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to [add a custom repo to your router](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router) first.
+If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to add a custom repo to your router following instructions on [GitHub](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router)/[jsDelivr](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/README.md#on-your-router) first.
 
 ## Default Settings
 
-Default configuration has service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM.
+Default configuration has the service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM.
 
 If your router has less then 64Mb RAM, edit the configuration file, located at ```/etc/config/simple-adblock```. The configuration file has lists in ascending order starting with smallest ones and each list has a preceding comment indicating its size, comment out or delete the lists you don't want or your router can't handle.
 
index 44bdc8e25dd7a9186ec538041fe3706be4d3a598..9bf8d6ea4024bc1b1bb83f0192e294c130d9d914 100644 (file)
@@ -11,7 +11,7 @@ config simple-adblock 'config'
        option parallel_downloads '1'
        option debug '0'
        option compressed_cache '0'
-       list whitelist_domain 'raw.githubusercontent.com'
+       list whitelist_domain 'cdn.jsdelivr.net'
 
 # Thu Oct  3 17:54:04 PDT 2019
 # File size: 4.0K
@@ -45,7 +45,7 @@ config simple-adblock 'config'
        list blacklist_hosts_url 'https://adaway.org/hosts.txt'
        
 # File size: 20.0K
-       list blacklist_hosts_url 'https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt'
+       list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@master/hosts.txt'
 
 # File size: 36.0K
        list blacklist_hosts_url 'https://www.malwaredomainlist.com/hostslist/hosts.txt'
@@ -56,7 +56,7 @@ config simple-adblock 'config'
 # File size: 388.0K
 # blocklist may be too big for some routers
 # blocklist may block some video-streaming content
-#      list blacklist_hosts_url 'https://raw.githubusercontent.com/jawz101/MobileAdTrackers/master/hosts'
+#      list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@master/hosts'
 
 # File size: 424.0K
 # blocklist may be too big for some routers
@@ -72,7 +72,7 @@ config simple-adblock 'config'
 
 # File size: 1.6M
 # blocklist too big for most routers
-#      list blacklist_hosts_url 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'
+#      list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/StevenBlack/hosts@master/hosts'
 
 # File size: 3.1M
 # blocklist too big for most routers
@@ -80,7 +80,7 @@ config simple-adblock 'config'
 
 # File size: 23.0M
 # blocklist too big for most routers
-#      list blacklist_hosts_url 'https://dbl.oisd.nl/'
+#      list blacklist_hosts_url 'https://hosts.oisd.nl/'
 
 # site was down on last check
 #      list blacklist_domains_url 'http://support.it-mate.co.uk/downloads/hosts.txt'
diff --git a/net/simple-adblock/files/simple-adblock.conf.update b/net/simple-adblock/files/simple-adblock.conf.update
new file mode 100644 (file)
index 0000000..1c1a5a1
--- /dev/null
@@ -0,0 +1,4 @@
+s|dbl.oisd.nl|hosts.oisd.nl|g
+s|raw.githubusercontent.com/StevenBlack/hosts/|cdn.jsdelivr.net/gh/StevenBlack/hosts@|g
+s|raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/|cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@|g
+s|raw.githubusercontent.com/jawz101/MobileAdTrackers/|cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@|g
index fad1d399d3a2df6b077161bd72acbddffd3b4a6e..883a6e9218c26a1746a0aff0ef243bb3dc752227 100644 (file)
@@ -145,6 +145,7 @@ export whitelist_domains blacklist_domains
 export whitelist_domains_urls blacklist_domains_urls blacklist_hosts_urls
 export wan_if wan_gw wanphysdev dl_command serviceStatus dl_flag
 export outputFilter outputFilterIPv6 outputFile outputGzip outputCache ipv6Enabled
+export is_ssl_supported
 
 load_package_config() {
        config_load "$packageName"
@@ -231,11 +232,11 @@ load_package_config() {
        fi
        . /lib/functions/network.sh
        . /usr/share/libubox/jshn.sh
-       # Prefer curl because it supports the file: scheme.
+       # Prefer curl because it supports the file:// scheme.
        if [ -x /usr/bin/curl ]; then
                dl_command="curl --insecure --retry $curlRetry --connect-timeout $dlTimeout --silent"
                dl_flag="-o"
-       elif wget -V 2>/dev/null | grep -q "+ssl"; then
+       elif wget --version 2>/dev/null | grep -q "+https"; then
                dl_command="wget --no-check-certificate --timeout $dlTimeout -q"
                dl_flag="-O"
        else
@@ -243,6 +244,15 @@ load_package_config() {
                dl_flag="-O"
        fi
        led="${led:+/sys/class/leds/$led}"
+       if curl --version 2>/dev/null | grep -q "https" \
+               || wget --version 2>/dev/null | grep -q "+https" \
+               || grep -q "libustream-mbedtls" /usr/lib/opkg/status \
+               || grep -q "libustream-openssl" /usr/lib/opkg/status \
+               || grep -q "libustream-wolfssl" /usr/lib/opkg/status; then
+               is_ssl_supported=1
+       else
+               unset is_ssl_supported
+       fi
 }
 
 is_enabled() {
@@ -668,19 +678,25 @@ process_url() {
        else
                type='Allowed'; D_TMP="$A_TMP";
        fi
+       if [ "${1:0:5}" == "https" ] && [ -z "$is_ssl_supported" ]; then
+               output 1 "$_FAIL_"
+               output 2 "[DL] $type $label $__FAIL__\\n"
+               echo "errorNoSSLSupport|${1}" >> "$sharedMemoryError"
+               return 0
+       fi
        while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do
                R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)"
        done
        if ! $dl_command "$1" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
                output 1 "$_FAIL_"
                output 2 "[DL] $type $label $__FAIL__\\n"
-               echo "errorDownloadingList=${1}" >> "$sharedMemoryError"
+               echo "errorDownloadingList|${1}" >> "$sharedMemoryError"
        else
                sed -i "$filter" "$R_TMP"
                if [ ! -s "$R_TMP" ]; then
                        output 1 "$_FAIL_"
                        output 2 "[DL] $type $label $__FAIL__\\n"
-                       echo "errorParsingList=${1}" >> "$sharedMemoryError"
+                       echo "errorParsingList|${1}" >> "$sharedMemoryError"
                else
                        cat "${R_TMP}" >> "$D_TMP"
                        output 1 "$_OK_"
@@ -1115,8 +1131,8 @@ showstatus() {
        fi
        if [ -n "$error" ]; then
                for c in $error; do
-                       url="${c##*=}"
-                       c="${c%=*}"
+                       url="${c##*|}"
+                       c="${c%|*}"
                        case "$c" in
                                errorDownloadingList|errorParsingList)
                                        output "$_ERROR_: $(getErrorText "$c") $url!\\n";;