From 901159b8c7127dc615d79451e3eb8cb6e9a51f75 Mon Sep 17 00:00:00 2001 From: Stan Grishin Date: Wed, 27 May 2020 00:25:57 +0000 Subject: [PATCH] simple-adblock: bugfix: proper error reporting on failed downloads; lists update script Signed-off-by: Stan Grishin --- net/simple-adblock/Makefile | 7 ++- net/simple-adblock/files/README.md | 51 +++++++++++-------- net/simple-adblock/files/simple-adblock.conf | 10 ++-- .../files/simple-adblock.conf.update | 4 ++ net/simple-adblock/files/simple-adblock.init | 28 +++++++--- 5 files changed, 65 insertions(+), 35 deletions(-) create mode 100644 net/simple-adblock/files/simple-adblock.conf.update diff --git a/net/simple-adblock/Makefile b/net/simple-adblock/Makefile index 793cb14d5a..2c45fe9ee2 100644 --- a/net/simple-adblock/Makefile +++ b/net/simple-adblock/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=simple-adblock PKG_VERSION:=1.8.3 -PKG_RELEASE:=3 +PKG_RELEASE:=9 PKG_MAINTAINER:=Stan Grishin PKG_LICENSE:=GPL-3.0-or-later @@ -21,7 +21,7 @@ endef define Package/simple-adblock/description This service provides DNSMASQ or Unbound based ad blocking. -Please see the project's README at github for further information. +Please see the project's README on GitHub for further information. endef @@ -46,6 +46,8 @@ define Package/simple-adblock/install $(INSTALL_BIN) $(PKG_BUILD_DIR)/files/simple-adblock.init $(1)/etc/init.d/simple-adblock $(INSTALL_DIR) $(1)/etc/config $(INSTALL_CONF) ./files/simple-adblock.conf $(1)/etc/config/simple-adblock + $(INSTALL_DIR) $(1)/tmp + $(INSTALL_CONF) ./files/simple-adblock.conf.update $(1)/tmp/simple-adblock.conf.update $(INSTALL_DIR) $(1)/etc/hotplug.d/iface $(INSTALL_BIN) ./files/simple-adblock.hotplug $(1)/etc/hotplug.d/iface/80-simple-adblock endef @@ -54,6 +56,7 @@ define Package/simple-adblock/postinst #!/bin/sh # check if we are on real system if [ -z "$${IPKG_INSTROOT}" ]; then + sed -f /tmp/simple-adblock.conf.update -i /etc/config/simple-adblock || true /etc/init.d/simple-adblock enable fi exit 0 diff --git a/net/simple-adblock/files/README.md b/net/simple-adblock/files/README.md index 8582f9e164..f5fd095a84 100644 --- a/net/simple-adblock/files/README.md +++ b/net/simple-adblock/files/README.md @@ -19,63 +19,70 @@ A simple DNSMASQ/Unbound-based AdBlocking service for OpenWrt/LEDE Project. - Blocks ads inside browsers with [DNS-over-HTTPS proxy](https://en.wikipedia.org/wiki/DNS_over_HTTPS) built-in, like [Mozilla Firefox](https://support.mozilla.org/en-US/kb/firefox-dns-over-https#w_about-dns-over-https) or [Google Chrome/Chromium](https://blog.chromium.org/2019/09/experimenting-with-same-provider-dns.html) -- with the ```dnsmasq.ipset``` option. - Proudly made in Canada, using locally-sourced electrons. -If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out [net/adblock](https://github.com/openwrt/packages/tree/master/net/adblock/files). +If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out ```net/adblock``` on [GitHub](https://github.com/openwrt/packages/tree/master/net/adblock/files)/[jsDelivr](https://cdn.jsdelivr.net/gh/openwrt/packages@master/net/adblock/files/README.md). ## Screenshots (luci-app-simple-adblock) Service Status -![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-status.png "Service Status") +![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-status.png "Service Status") Configuration - Basic Configuration -![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-config-basic.png "Configuration - Basic Configuration") +![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-config-basic.png "Configuration - Basic Configuration") Configuration - Advanced Configuration -![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-config-advanced.png "Configuration - Advanced Configuration") +![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-config-advanced.png "Configuration - Advanced Configuration") Whitelist and Blocklist Management -![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-lists.png "Whitelist and Blocklist Management") +![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-lists.png "Whitelist and Blocklist Management") ## Requirements -This service requires the following packages to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound``` and either ```ca-certificates```, ```wget``` and ```libopenssl``` (for OpenWrt 15.05.1) or ```uclient-fetch``` and ```libustream-mbedtls``` (for LEDE Project and OpenWrt 18.06.xx or newer). Additionally installation of ```coreutils-sort``` is highly recommended as it speeds up blocklist processing. +To download the hosts/domains files this service requires one of the following packages to be installed on your device: ```wget``` or ```uclient-fetch``` or ```curl```. If you want to download lists over HTTPS (recommended), you also need to install support for HTTPS/SSL downloads on your device. For most devices starting with OpenWrt 15.05.1 and up the following commands will install the minimal requirements: -To satisfy the requirements for connect to your router via ssh and run the following commands: +```sh +opkg update; if ubus -S call system board | grep -q '15.05'; then +opkg install ca-certificates wget libopenssl; else +opkg install uclient-fetch libustream-mbedtls ca-bundle ca-certificates; fi +``` -### How to use DNSMASQ ipset +### Requirements for file:// Scheme -The ```dnsmasq.ipset``` option requires you to install ```ipset``` and ```dnsmasq-full``` instead of the ```dnsmasq```. To do that, connect to your router via ssh and run the following command: +If you want to include some local files as the blocklists or whitelists, you need to install ```curl```: ```sh -opkg update; opkg remove dnsmasq; opkg install dnsmasq-full ipset; +opkg update; opkg install curl; ``` -### OpenWrt 15.05.1 Requirements +### Requirements for DNS Resolver -```sh -opkg update; opkg install ca-certificates wget libopenssl coreutils-sort dnsmasq -``` +In order to actually block the ads, this service requires one of the following DNS resolvers to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound```. The ```dnsmasq``` package is usually included in the default OpenWrt installation. -### LEDE Project 17.01.x and OpenWrt 18.xx (or newer) Requirements +If you want to use the [```dnsmasq.ipset``` option](#dns-resolution-option) you need to install ```ipset``` and ```dnsmasq-full``` instead of the ```dnsmasq```. To do that, connect to your router via ssh and run the following command: ```sh -opkg update; opkg install uclient-fetch libustream-mbedtls coreutils-sort dnsmasq +opkg update; cd /tmp/; opkg download dnsmasq-full; opkg install ipset; +opkg remove dnsmasq; opkg install /tmp/dnsmasq-full*; rm -f /tmp/dnsmasq-full*; ``` -### IPv6 Support +### Requirements for IPv6 Support For IPv6 support additionally install ```ip6tables-mod-nat``` and ```kmod-ipt-nat6``` packages from Web UI or run the following in the command line: ```sh -opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6 +opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6; ``` -### Speed Up Blocklist Processing +### Requirements for Faster Blocklist Processing -The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run ```opkg --force-overwrite install coreutils-sort```. +The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run the following command: + +```sh +opkg --force-overwrite install coreutils-sort +``` ## Unmet Dependencies @@ -89,11 +96,11 @@ Install ```simple-adblock``` and ```luci-app-simple-adblock``` packages from Web opkg update; opkg install simple-adblock luci-app-simple-adblock ``` -If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to [add a custom repo to your router](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router) first. +If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to add a custom repo to your router following instructions on [GitHub](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router)/[jsDelivr](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/README.md#on-your-router) first. ## Default Settings -Default configuration has service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM. +Default configuration has the service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM. If your router has less then 64Mb RAM, edit the configuration file, located at ```/etc/config/simple-adblock```. The configuration file has lists in ascending order starting with smallest ones and each list has a preceding comment indicating its size, comment out or delete the lists you don't want or your router can't handle. diff --git a/net/simple-adblock/files/simple-adblock.conf b/net/simple-adblock/files/simple-adblock.conf index 44bdc8e25d..9bf8d6ea40 100644 --- a/net/simple-adblock/files/simple-adblock.conf +++ b/net/simple-adblock/files/simple-adblock.conf @@ -11,7 +11,7 @@ config simple-adblock 'config' option parallel_downloads '1' option debug '0' option compressed_cache '0' - list whitelist_domain 'raw.githubusercontent.com' + list whitelist_domain 'cdn.jsdelivr.net' # Thu Oct 3 17:54:04 PDT 2019 # File size: 4.0K @@ -45,7 +45,7 @@ config simple-adblock 'config' list blacklist_hosts_url 'https://adaway.org/hosts.txt' # File size: 20.0K - list blacklist_hosts_url 'https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt' + list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@master/hosts.txt' # File size: 36.0K list blacklist_hosts_url 'https://www.malwaredomainlist.com/hostslist/hosts.txt' @@ -56,7 +56,7 @@ config simple-adblock 'config' # File size: 388.0K # blocklist may be too big for some routers # blocklist may block some video-streaming content -# list blacklist_hosts_url 'https://raw.githubusercontent.com/jawz101/MobileAdTrackers/master/hosts' +# list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@master/hosts' # File size: 424.0K # blocklist may be too big for some routers @@ -72,7 +72,7 @@ config simple-adblock 'config' # File size: 1.6M # blocklist too big for most routers -# list blacklist_hosts_url 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts' +# list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/StevenBlack/hosts@master/hosts' # File size: 3.1M # blocklist too big for most routers @@ -80,7 +80,7 @@ config simple-adblock 'config' # File size: 23.0M # blocklist too big for most routers -# list blacklist_hosts_url 'https://dbl.oisd.nl/' +# list blacklist_hosts_url 'https://hosts.oisd.nl/' # site was down on last check # list blacklist_domains_url 'http://support.it-mate.co.uk/downloads/hosts.txt' diff --git a/net/simple-adblock/files/simple-adblock.conf.update b/net/simple-adblock/files/simple-adblock.conf.update new file mode 100644 index 0000000000..1c1a5a1c50 --- /dev/null +++ b/net/simple-adblock/files/simple-adblock.conf.update @@ -0,0 +1,4 @@ +s|dbl.oisd.nl|hosts.oisd.nl|g +s|raw.githubusercontent.com/StevenBlack/hosts/|cdn.jsdelivr.net/gh/StevenBlack/hosts@|g +s|raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/|cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@|g +s|raw.githubusercontent.com/jawz101/MobileAdTrackers/|cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@|g diff --git a/net/simple-adblock/files/simple-adblock.init b/net/simple-adblock/files/simple-adblock.init index fad1d399d3..883a6e9218 100644 --- a/net/simple-adblock/files/simple-adblock.init +++ b/net/simple-adblock/files/simple-adblock.init @@ -145,6 +145,7 @@ export whitelist_domains blacklist_domains export whitelist_domains_urls blacklist_domains_urls blacklist_hosts_urls export wan_if wan_gw wanphysdev dl_command serviceStatus dl_flag export outputFilter outputFilterIPv6 outputFile outputGzip outputCache ipv6Enabled +export is_ssl_supported load_package_config() { config_load "$packageName" @@ -231,11 +232,11 @@ load_package_config() { fi . /lib/functions/network.sh . /usr/share/libubox/jshn.sh - # Prefer curl because it supports the file: scheme. + # Prefer curl because it supports the file:// scheme. if [ -x /usr/bin/curl ]; then dl_command="curl --insecure --retry $curlRetry --connect-timeout $dlTimeout --silent" dl_flag="-o" - elif wget -V 2>/dev/null | grep -q "+ssl"; then + elif wget --version 2>/dev/null | grep -q "+https"; then dl_command="wget --no-check-certificate --timeout $dlTimeout -q" dl_flag="-O" else @@ -243,6 +244,15 @@ load_package_config() { dl_flag="-O" fi led="${led:+/sys/class/leds/$led}" + if curl --version 2>/dev/null | grep -q "https" \ + || wget --version 2>/dev/null | grep -q "+https" \ + || grep -q "libustream-mbedtls" /usr/lib/opkg/status \ + || grep -q "libustream-openssl" /usr/lib/opkg/status \ + || grep -q "libustream-wolfssl" /usr/lib/opkg/status; then + is_ssl_supported=1 + else + unset is_ssl_supported + fi } is_enabled() { @@ -668,19 +678,25 @@ process_url() { else type='Allowed'; D_TMP="$A_TMP"; fi + if [ "${1:0:5}" == "https" ] && [ -z "$is_ssl_supported" ]; then + output 1 "$_FAIL_" + output 2 "[DL] $type $label $__FAIL__\\n" + echo "errorNoSSLSupport|${1}" >> "$sharedMemoryError" + return 0 + fi while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)" done if ! $dl_command "$1" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then output 1 "$_FAIL_" output 2 "[DL] $type $label $__FAIL__\\n" - echo "errorDownloadingList=${1}" >> "$sharedMemoryError" + echo "errorDownloadingList|${1}" >> "$sharedMemoryError" else sed -i "$filter" "$R_TMP" if [ ! -s "$R_TMP" ]; then output 1 "$_FAIL_" output 2 "[DL] $type $label $__FAIL__\\n" - echo "errorParsingList=${1}" >> "$sharedMemoryError" + echo "errorParsingList|${1}" >> "$sharedMemoryError" else cat "${R_TMP}" >> "$D_TMP" output 1 "$_OK_" @@ -1115,8 +1131,8 @@ showstatus() { fi if [ -n "$error" ]; then for c in $error; do - url="${c##*=}" - c="${c%=*}" + url="${c##*|}" + c="${c%|*}" case "$c" in errorDownloadingList|errorParsingList) output "$_ERROR_: $(getErrorText "$c") $url!\\n";; -- 2.30.2