From: Stan Grishin Date: Sat, 17 Aug 2019 13:09:43 +0000 (-0700) Subject: [18.06] simple-adblock: support for varios DNS resolvers/options X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=8e8087706d64e2953abfe7d4747d2b4f242d4cad;p=feed%2Fpackages.git [18.06] simple-adblock: support for varios DNS resolvers/options Signed-off-by: Stan Grishin --- diff --git a/net/simple-adblock/Makefile b/net/simple-adblock/Makefile index 81081ab4fc..4e8024230c 100644 --- a/net/simple-adblock/Makefile +++ b/net/simple-adblock/Makefile @@ -5,8 +5,8 @@ include $(TOPDIR)/rules.mk PKG_NAME:=simple-adblock -PKG_VERSION:=1.7.0 -PKG_RELEASE:=1 +PKG_VERSION:=1.8.0 +PKG_RELEASE:=0 PKG_MAINTAINER:=Stan Grishin PKG_LICENSE:=GPL-3.0-or-later @@ -20,8 +20,8 @@ define Package/simple-adblock endef define Package/simple-adblock/description -This service provides dnsmasq-based ad blocking. -Please see the README for further information. +This service provides DNSMASQ or Unbound based ad blocking. +Please see the project's README at github for further information. endef @@ -71,6 +71,10 @@ define Package/simple-adblock/prerm if [ -z "$${IPKG_INSTROOT}" ]; then while uci -q del ucitrack.@simple-adblock[-1]; do :; done echo "Stopping service and removing rc.d symlink for simple-adblock" + uci -q del_list dhcp.@dnsmasq[0].addnhosts="/var/run/simple-adblock.addnhosts" || true + if [ "$$(uci -q get dhcp.@dnsmasq[0].serversfile)" == "/var/run/simple-adblock.servers" ]; then + uci -q del dhcp.@dnsmasq[0].serversfile || true + fi /etc/init.d/simple-adblock stop || true /etc/init.d/simple-adblock killcache || true /etc/init.d/simple-adblock disable || true diff --git a/net/simple-adblock/files/README.md b/net/simple-adblock/files/README.md index 605e30a1ec..ad311a2201 100644 --- a/net/simple-adblock/files/README.md +++ b/net/simple-adblock/files/README.md @@ -1,34 +1,32 @@ # Simple AdBlock -A simple DNSMASQ-based AdBlocking service for OpenWrt/LEDE Project. Loosely based on [bole5's](https://forum.openwrt.org/profile.php?id=45571) idea with major performance improvements, added features and Web UI (as a separate package); inspired by @dibdot's innovation. +A simple DNSMASQ/Unbound-based AdBlocking service for OpenWrt/LEDE Project. ## Features -- Supports OpenWrt Designated Driver and LEDE Project. -- Super-fast due to the nature of supported block lists and backgrounding of already downloaded data while next list is downloading. +- Super-fast due to the nature of supported block lists and parallel downloading/processing of the blacklists. - Supports both hosts files and domains lists for blocking (to keep it lean and fast). - Everything is configurable from Web UI. - Allows you to easily add your own domains to whitelist or blacklist. - Allows you to easily add URLs to your own blocked hosts or domains lists to block/whitelist (just put whitelisted domains one per line). - Requires no configuration for the download utility wherever you want to use wget/libopenssl or uclient-fetch/libustream-mbedtls. -- Installs dependencies automatically (DD/LEDE-default uclient-fetch libustream-mbedtls). -- Doesn't stay in memory -- creates the list of blocked domains and then uses DNSMASQ and firewall rules to serve "domain not found reply". -- As some of the default lists are using https, reliably works with either wget/libopenssl or uclient-fetch/libustream-mbedtls. -- Very lightweight and easily hackable, the whole script is just one /etc/init.d/simple-adblock file. -- Logs single entry in the system log with the number of blocked domains if verbosity is set to 0. -- Retains the downloaded/sorted adblocking list on service stop and reuses it on service start (use reload if you want to force re-download of the list). -- Blocks ads served over https. +- Installs dependencies automatically. +- Doesn't stay in memory -- creates the list of blocked domains and then uses DNSMASQ/Unbound and firewall rules to serve NXDOMAIN or 127.0.0.1 (depending on settings) reply for blocked domains. +- As some of the default lists are using https, reliably works with either wget/libopenssl, uclient-fetch/libustream-mbedtls or curl. +- Very lightweight and easily hackable, the whole script is just one ```/etc/init.d/simple-adblock``` file. +- Retains the downloaded/sorted AdBlocking list on service stop and reuses it on service start (use ```dl``` command if you want to force re-download of the list). +- Blocks ads served over https (unlike PixelServ-derived solutions). - Proudly made in Canada, using locally-sourced electrons. -If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out [@dibdot's adblock](https://github.com/openwrt/packages/tree/master/net/adblock/files). +If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out [net/adblock](https://github.com/openwrt/packages/tree/master/net/adblock/files). ## Screenshot (luci-app-simple-adblock) -![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot06.png "screenshot") +![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot07.png "screenshot") ## Requirements -This service requires the following packages to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` and either ```ca-certificates```, ```wget``` and ```libopenssl``` (for OpenWrt 15.05.1) or ```uclient-fetch``` and ```libustream-mbedtls``` (for OpenWrt DD trunk and all LEDE Project builds). Additionally installation of ```coreutils-sort``` is highly recommended as it speeds up blocklist processing. +This service requires the following packages to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound``` and either ```ca-certificates```, ```wget``` and ```libopenssl``` (for OpenWrt 15.05.1) or ```uclient-fetch``` and ```libustream-mbedtls``` (for LEDE Project and OpenWrt 18.06.xx or newer). Additionally installation of ```coreutils-sort``` is highly recommended as it speeds up blocklist processing. To satisfy the requirements for connect to your router via ssh and run the following commands: @@ -38,7 +36,7 @@ To satisfy the requirements for connect to your router via ssh and run the follo opkg update; opkg install ca-certificates wget libopenssl coreutils-sort dnsmasq ``` -### LEDE Project 17.01.x and OpenWrt 18.xx or later Requirements +### LEDE Project 17.01.x and OpenWrt 18.xx (or newer) Requirements ```sh opkg update; opkg install uclient-fetch libustream-mbedtls coreutils-sort dnsmasq @@ -52,15 +50,15 @@ For IPv6 support additionally install ```ip6tables-mod-nat``` and ```kmod-ipt-na opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6 ``` -### Speed up blocklist processing with coreutils-sort +### Speed Up Blocklist Processing The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run ```opkg --force-overwrite install coreutils-sort```. -## Unmet dependencies +## Unmet Dependencies If you are running a development (trunk/snapshot) build of OpenWrt/LEDE Project on your router and your build is outdated (meaning that packages of the same revision/commit hash are no longer available and when you try to satisfy the [requirements](#requirements) you get errors), please flash either current LEDE release image or current development/snapshot image. -## How to install +## How To Install Install ```simple-adblock``` and ```luci-app-simple-adblock``` packages from Web UI or run the following in the command line: @@ -68,63 +66,72 @@ Install ```simple-adblock``` and ```luci-app-simple-adblock``` packages from Web opkg update; opkg install simple-adblock luci-app-simple-adblock ``` -If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to [add a custom repo to your router](#add-custom-repo-to-your-router) first. - -### Add custom repo to your router - -If your router is not set up with the access to repository containing these packages you will need to add custom repository to your router by connecting to your router via ssh and running the following commands: - -#### OpenWrt 15.05.1 Instructions - -```sh -opkg update; opkg install ca-certificates wget libopenssl -echo -e -n 'untrusted comment: LEDE usign key of Stan Grishin\nRWR//HUXxMwMVnx7fESOKO7x8XoW4/dRidJPjt91hAAU2L59mYvHy0Fa\n' > /tmp/stangri-repo.pub && opkg-key add /tmp/stangri-repo.pub -! grep -q 'stangri_repo' /etc/opkg/customfeeds.conf && echo 'src/gz stangri_repo https://raw.githubusercontent.com/stangri/openwrt-repo/master' >> /etc/opkg/customfeeds.conf -opkg update -opkg install simple-adblock luci-app-simple-adblock -``` - -#### LEDE Project 17.01.x and OpenWrt 18.06.x Instructions - -```sh -opkg update -opkg list-installed | grep -q uclient-fetch || opkg install uclient-fetch -opkg list-installed | grep -q libustream || opkg install libustream-mbedtls -echo -e -n 'untrusted comment: LEDE usign key of Stan Grishin\nRWR//HUXxMwMVnx7fESOKO7x8XoW4/dRidJPjt91hAAU2L59mYvHy0Fa\n' > /tmp/stangri-repo.pub && opkg-key add /tmp/stangri-repo.pub -! grep -q 'stangri_repo' /etc/opkg/customfeeds.conf && echo 'src/gz stangri_repo https://raw.githubusercontent.com/stangri/openwrt-repo/master' >> /etc/opkg/customfeeds.conf -opkg update -opkg install simple-adblock luci-app-simple-adblock -``` +If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to [add a custom repo to your router](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router) first. ## Default Settings -Default configuration has service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1```) and selected ad/malware lists suitable for routers with 64Mb RAM. The configuration file has lists in descending order starting with biggest ones, comment out or delete the lists you don't want or your router can't handle. +Default configuration has service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM. The configuration file has lists in descending order starting with biggest ones, comment out or delete the lists you don't want or your router can't handle. -## How to customize +## How To Customize You can use Web UI (found in Services/Simple AdBlock) to add/remove/edit links to: -- hosts files (127.0.0.1 or 0.0.0.0 followed by space and domain name per line) to be blocked. +- [hosts files](https://en.wikipedia.org/wiki/Hosts_(file)) (127.0.0.1 or 0.0.0.0 followed by space and domain name per line) to be blocked. - domains lists (one domain name per line) to be blocked. -- domains lists (one domain name per line) to be whitelisted. It is useful if you want to run simple-adblock on multiple routers and maintain one centralized whitelist which you can publish on a web-server. +- domains lists (one domain name per line) to be whitelisted. It is useful if you want to run ```simple-adblock``` on multiple routers and maintain one centralized whitelist which you can publish on a web-server. -Please note that these lists **have** to include either ```http://``` or ```https://``` (or, if ```curl``` is installed the ```file://```) prefix. Some of the top block lists (both hosts files and domains lists) suitable for routers with at least 8MB RAM are used in the default simple-adblock installation. +Please note that these lists **must** include either ```http://``` or ```https://``` (or, if ```curl``` is installed the ```file://```) prefix. Some of the top block lists (both hosts files and domains lists) suitable for routers with at least 8MB RAM are used in the default ```simple-adblock``` installation. You can also use Web UI to add individual domains to be blocked or whitelisted. -If you want to use CLI to customize simple-adblock config, you can probably figure out how to do it by looking at the contents of ```/etc/config/simple-adblock``` or output of the ```uci show simple-adblock``` command. +If you want to use CLI to customize ```simple-adblock``` config, refer to the [Customization Settings](#customization-settings) section. -## How to use +## How To Use Once the service is enabled in the [config file](#default-settings), run ```/etc/init.d/simple-adblock start``` to start the service. Either ```/etc/init.d/simple-adblock restart``` or ```/etc/init.d/simple-adblock reload``` will only restart the service and/or re-donwload the lists if there were relevant changes in the config file since the last successful start. Had the previous start resulted in any error, either ```/etc/init.d/simple-adblock start```, ```/etc/init.d/simple-adblock restart``` or ```/etc/init.d/simple-adblock reload``` will attempt to re-download the lists. -If you want to force simple-adblock to re-download the lists, run ```/etc/init.d/simple-adblock download```. +If you want to force simple-adblock to re-download the lists, run ```/etc/init.d/simple-adblock dl```. If you want to check if the specific domain (or part of the domain name) is being blocked, run ```/etc/init.d/simple-adblock check test-domain.com```. -## How does it work - -This service downloads (and processes in the background, removing comments and other useless data) lists of hosts and domains to be blocked, combines those lists into one big block list, removes duplicates and sorts it and then removes your whitelisted domains from the block list before converting to to dnsmasq-compatible file and restarting dnsmasq. The result of the process is that dnsmasq returns "domain not found" for the blocked domains. +## Configuration Settings + +In the Web UI the ```simple-adblock``` settings are split into ```basic``` and ```advanced``` settings. The full list of configuration parameters of ```simple-adblock.config``` section is: + +|Web UI Section|Parameter|Type|Default|Description| +| --- | --- | --- | --- | --- | +|Basic|enabled|boolean|0|Enable/disable the ```simple-adblock``` service.| +|Basic|verbosity|integer|2|Can be set to 0, 1 or 2 to control the console and system log output verbosity of the ```simple-adblock``` service.| +|Basic|force_dns|boolean|1|Force router's DNS to local devices which may have different/hardcoded DNS server settings. If enabled, creates a firewall rule to intercept DNS requests from local devices to external DNS servers and redirect them to router.| +|Basic|led|string|none|Use one of the router LEDs to indicate the AdBlocking status.| +|Advanced|dns|string|dnsmasq.servers|DNS resolution option. See [table below](#dns-resolution-option) for addtional information.| +|Advanced|ipv6_enabled|boolean|0|Add IPv6 entries to block-list if ```dnsmasq.addnhosts``` is used. This option is only visible in Web UI if the ```dnsmasq.addnhosts``` is selected as the DNS resolution option.| +|Advanced|boot_delay|integer|120|Delay service activation for that many seconds on boot up. You can shorten it to 10-30 seconds on modern fast routers. Routers with built-in modems may require longer boot delay.| +|Advanced|download_timeout|integer|10|Time-out downloads if no reply received within that many last seconds.| +|Advanced|curl_retry|integer|3|If ```curl``` is installed and detected, attempt that many retries for failed downloads.| +|Advanced|parallel_downloads|boolean|1|If enabled, all downloads are completed concurrently, if disabled -- sequentioally. Concurrent downloads dramatically speed up service loading.| +|Advanced|debug|boolean|0|If enabled, output service full debug to ```/tmp/simple-adblock.log```. Please note that the debug file may clog up the router's RAM on some devices. Use with caution.| +|Advanced|allow_non_ascii|boolean|0|Enable support for non-ASCII characters in the final AdBlocking file. Only enable if your target service supports non-ASCII characters. If you enable this on the system where DNS resolver doesn't support non-ASCII characters, it will crash. Use with caution.| +|Advanced|compressed_cache|boolean|0|Create compressed cache of the AdBlocking file in router's persistent memory. Only recommended to be used on routers with large ROM and/or routers with metered/flaky internet connection.| +||whitelist_domain|list/string||List of white-listed domains.| +||whitelist_domains_url|list/string||List of URL(s) to text files containing white-listed domains. **Must** include either ```http://``` or ```https://``` (or, if ```curl``` is installed the ```file://```) prefix. Useful if you want to keep/publish a single white-list for multiple routers.| +||blacklist_domains_url|list/string||List of URL(s) to text files containing black-listed domains. **Must** include either ```http://``` or ```https://``` (or, if ```curl``` is installed the ```file://```) prefix.| +||blacklist_hosts_url|list/string||List of URL(s) to [hosts files](https://en.wikipedia.org/wiki/Hosts_(file)) containing black-listed domains. **Must** include either ```http://``` or ```https://``` (or, if ```curl``` is installed the ```file://```) prefix.| + +### DNS Resolution Option + +Currently supported options are: + +|Option|Explanation| +| --- | --- | +|```dnsmasq.addnhosts```|Creates the DNSMASQ additional hosts file ```/var/run/simple-adblock.addnhosts``` and modifies DNSMASQ settings, so that DNSMASQ resolves all blocked domains to "local machine": 127.0.0.1. This option doesn't allow block-list optimization (by removing secondary level domains if the top-level domain is also in the block-list), so it results in a much larger block-list file, but, unlike other DNSMASQ-based options, it has almost no effect on the DNS look up speed. This option also allows quick reloads of DNSMASQ on block-list updates.| +|```dnsmasq.conf```|Creates the DNSMASQ config file ```/var/dnsmasq.d/simple-adblock``` so that DNSMASQ replies with NXDOMAIN: "domain not found". This option allows the block-list optimization (by removing secondary level domains if the top-level domain is also in the block-list), resulting in the smaller block-list file. This option will slow down DNS look up speed somewhat.| +|```dnsmasq.servers```|Creates the DNSMASQ servers file ```/var/run/simple-adblock.servers``` and modifies DNSMASQ settings so that DNSMASQ replies with NXDOMAIN: "domain not found". This option allows the block-list optimization (by removing secondary level domains if the top-level domain is also in the block-list), resulting in the smaller block-list file. This option will slow down DNS look up speed somewhat. This is a default setting as it results in the smaller block-file and allows quick reloads of DNSMASQ.| +|```unbound.adb_list```|Creates the Unbound config file ```/var/lib/unbound/adb_list.simple-adblock``` so that Unbound replies with NXDOMAIN: "domain not found". This option allows the block-list optimization (by removing secondary level domains if the top-level domain is also in the block-list), resulting in the smaller block-list file.| + +## How Does It Work + +This service downloads (and processes in the background, removing comments and other useless data) lists of hosts and domains to be blocked, combines those lists into one big block list, removes duplicates and sorts it and then removes your whitelisted domains from the block list before converting to to DNSMASQ/Unbound-compatible file and restarting DNSMASQ/Unbound if needed. The result of the process is that DNSMASQ/Unbound return NXDOMAIN or 127.0.0.1 (depending on settings) for the blocked domains. If you specify ```google.com``` as a domain to be whitelisted, you will have access to ```google.com```, ```www.google.com```, ```analytics.google.com```, but not fake domains like ```email-google.com``` or ```drive.google.com.verify.signin.normandeassociation.com``` for example. If you only want to allow ```www.google.com``` while blocking all other ```google.com``` subdomains, just specify ```www.google.com``` as domain to be whitelisted. @@ -132,8 +139,8 @@ In general, whatever domain is specified to be whitelisted; it, along with with ## Documentation / Discussion -Please head [LEDE Project Forum](https://forum.lede-project.org/t/simple-adblock-fast-lean-and-fully-uci-luci-configurable-adblocking/1327/) for discussion of this package. +Please head to [OpenWrt Forum](https://forum.openwrt.org/t/simple-adblock-fast-lean-and-fully-uci-luci-configurable-adblocking/1327/) for discussion of this package. ## Thanks -I'd like to thank everyone who helped create, test and troubleshoot this service. Special thanks to [@hnyman](https://github.com/hnyman) for general package/luci guidance, [@dibdot](https://github.com/dibdot) for general guidance and block-list optimization code and [@ckuethe](https://github.com/ckuethe) for the curl support, non-ASCII filtering and compressed cache code. \ No newline at end of file +I'd like to thank everyone who helped create, test and troubleshoot this service. Special thanks to [@hnyman](https://github.com/hnyman) for general package/luci guidance, [@dibdot](https://github.com/dibdot) for general guidance and block-list optimization code, [@ckuethe](https://github.com/ckuethe) for the curl support, non-ASCII filtering and compressed cache code, [@EricLuehrsen](https://github.com/EricLuehrsen) for the Unbound support information and [@phasecat](https://forum.openwrt.org/u/phasecat/summary) for submitting bugs and testing. diff --git a/net/simple-adblock/files/simple-adblock.conf b/net/simple-adblock/files/simple-adblock.conf index 27a2a52fb2..a23f185041 100644 --- a/net/simple-adblock/files/simple-adblock.conf +++ b/net/simple-adblock/files/simple-adblock.conf @@ -1,10 +1,12 @@ config simple-adblock 'config' option enabled '0' + option dns 'dnsmasq.servers' option verbosity '2' option force_dns '1' option led 'none' option boot_delay '120' option download_timeout '10' + option curl_retry '3' option parallel_downloads '1' option debug '0' option allow_non_ascii '0' @@ -14,13 +16,12 @@ config simple-adblock 'config' # list blacklist_hosts_url 'https://hostsfile.mine.nu/Hosts' # list blacklist_hosts_url 'https://hosts-file.net/ad_servers.txt' # list blacklist_hosts_url 'http://sysctl.org/cameleon/hosts' - list blacklist_hosts_url 'http://www.mvps.org/winhelp2002/hosts.txt' + list blacklist_hosts_url 'http://winhelp2002.mvps.org/hosts.txt' list blacklist_hosts_url 'https://pgl.yoyo.org/as/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext' list blacklist_hosts_url 'https://www.malwaredomainlist.com/hostslist/hosts.txt' list blacklist_hosts_url 'https://adaway.org/hosts.txt' list blacklist_hosts_url 'https://someonewhocares.org/hosts/hosts' list blacklist_hosts_url 'https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt' - list blacklist_hosts_url 'https://zeustracker.abuse.ch/blocklist.php?download=hostfile' list blacklist_domains_url 'https://mirror1.malwaredomains.com/files/justdomains' list blacklist_domains_url 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt' list blacklist_domains_url 'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt' diff --git a/net/simple-adblock/files/simple-adblock.init b/net/simple-adblock/files/simple-adblock.init index 78cb6f4c61..9818bc6614 100644 --- a/net/simple-adblock/files/simple-adblock.init +++ b/net/simple-adblock/files/simple-adblock.init @@ -7,21 +7,34 @@ export LC_ALL=C export EXTRA_COMMANDS="check dl killcache status" export EXTRA_HELP=" check Checks if specified domain is found in current blacklist - dl Force-redownloads all the lists, even if the last download was successful and no config changes were made - status Shows the service last-run status" + dl Force-redownloads all the list + status Shows the service last-run status" readonly packageName="simple-adblock" readonly serviceName="$packageName $PKG_VERSION" +readonly addnhostsFile="/var/run/${packageName}.addnhosts" +readonly addnhostsCache="/var/run/${packageName}.addnhosts.cache" +readonly addnhostsGzip="/etc/${packageName}.addnhosts.gz" +readonly addnhostsOutputFilter='s|^|127.0.0.1 |;s|$||' +readonly addnhostsOutputFilterIPv6='s|^|:: |;s|$||' readonly dnsmasqFile="/var/dnsmasq.d/${packageName}" -readonly compressedCacheFile="/etc/${packageName}.gz" -readonly A_TMP="/var/simple-adblock.hosts.a.tmp" -readonly B_TMP="/var/simple-adblock.hosts.b.tmp" -readonly cacheFile="/var/run/${packageName}.cache" +readonly dnsmasqCache="/var/run/${packageName}.dnsmasq.cache" +readonly dnsmasqGzip="/etc/${packageName}.dnsmasq.gz" +readonly dnsmasqOutputFilter='s|^|local=/|;s|$|/|' +readonly serversFile="/var/run/${packageName}.servers" +readonly serversCache="/var/run/${packageName}.servers.cache" +readonly serversGzip="/etc/${packageName}.servers.gz" +readonly serversOutputFilter='s|^|server=/|;s|$|/|' +readonly unboundFile="/var/lib/unbound/adb_list.${packageName}" +readonly unboundCache="/var/run/${packageName}.unbound.cache" +readonly unboundGzip="/etc/${packageName}.unbound.gz" +readonly unboundOutputFilter='s|^|local-zone: "|;s|$|" static|' +readonly A_TMP="/var/${packageName}.hosts.a.tmp" +readonly B_TMP="/var/${packageName}.hosts.b.tmp" readonly PIDFile="/var/run/${packageName}.pid" -readonly JsonFile="/var/run/${packageName}.json" -readonly h_filter='/localhost/d;/^#/d;/^[^0-9]/d;s/^0\.0\.0\.0.//;s/^127\.0\.0\.1.//;s/[[:space:]]*#.*$//;s/[[:cntrl:]]$//;s/[[:space:]]//g;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;' -readonly d_filter='/^#/d;s/[[:space:]]*#.*$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;' -readonly f_filter='s|^|local=/|;s|$|/|' +readonly jsonFile="/var/run/${packageName}.json" +readonly hostsFilter='/localhost/d;/^#/d;/^[^0-9]/d;s/^0\.0\.0\.0.//;s/^127\.0\.0\.1.//;s/[[:space:]]*#.*$//;s/[[:cntrl:]]$//;s/[[:space:]]//g;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;' +readonly domainsFilter='/^#/d;s/[[:space:]]*#.*$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;' readonly checkmark='\xe2\x9c\x93' readonly xmark='\xe2\x9c\x97' readonly _OK_='\033[0;32m\xe2\x9c\x93\033[0m' @@ -39,19 +52,19 @@ readonly statusForceReloading='Force-Reloading' readonly statusProcessing='Processing' readonly statusStopped='Stopped' +export logmsg create_lock() { [ -e "$PIDFile" ] && return 1; touch "$PIDFile"; } -remove_lock() { [ -e "$PIDFile" ] && rm -f "$PIDFile"; } # rm -f /var/simple-adblock_tmp_* >/dev/null 2>&1; } +remove_lock() { [ -e "$PIDFile" ] && rm -f "$PIDFile"; } trap remove_lock EXIT -output_ok() { case $verbosity in 1) output 1 "$_OK_";; 2) output 2 "$__OK__\\n";; esac; } -output_okn() { case $verbosity in 1) output 1 "$_OK_\\n";; 2) output 2 "$__OK__\\n";; esac; } -output_fail() { case $verbosity in 1) output 1 "$_FAIL_";; 2) output 2 "$__FAIL__\\n";; esac; } -output_failn() { case $verbosity in 1) output 1 "$_FAIL_\\n";; 2) output 2 "$__FAIL__\\n";; esac; } -export logmsg +output_ok() { output 1 "$_OK_"; output 2 "$__OK__\\n"; } +output_okn() { output 1 "$_OK_\\n"; output 2 "$__OK__\\n"; } +output_fail() { output 1 "$_FAIL_\\n"; output 2 "$__FAIL__\\n"; } +output_failn() { output 1 "$_FAIL_"; output 2 "$__FAIL__\\n"; } output() { # Can take a single parameter (text) to be output at any verbosity # Or target verbosity level and text to be output at specifc verbosity if [ $# -ne 1 ]; then - if [ ! $((verbosity & $1)) -gt 0 ]; then return 0; else shift; fi + if [ $((verbosity & $1)) -gt 0 ] || [ "$verbosity" == "$1" ]; then shift; else return 0; fi fi [ -t 1 ] && echo -e -n "$1" local msg=$(echo -n "${1/$serviceName /service }" | sed 's|\\033\[[0-9]\?;\?[0-9]\?[0-9]\?m||g'); @@ -64,8 +77,6 @@ output() { } led_on(){ if [ -n "${1}" ] && [ -e "${1}/trigger" ]; then echo "default-on" > "${1}/trigger" 2>&1; fi; } led_off(){ if [ -n "${1}" ] && [ -e "${1}/trigger" ]; then echo "none" > "${1}/trigger" 2>&1; fi; } -testCompressedCache(){ [ "$compressedCache" -gt 0 ] && gzip -t -c $compressedCacheFile; } -boot() { load_package_config; ( sleep "$bootDelay" && rc_procd start_service && rc_procd service_triggers | cat & ); } export serviceEnabled export forceDNS @@ -73,8 +84,10 @@ export parallelDL export debug export allowNonAscii export compressedCache +export targetDNS export bootDelay export dlTimeout +export curlRetry export verbosity export led export whitelist_domains @@ -83,7 +96,7 @@ export whitelist_domains_urls export blacklist_domains_urls export blacklist_hosts_urls export wan_if wan_gw wanphysdev dl_command serviceStatus dl_flag -export dlStatus +export outputFilter outputFilterIPv6 outputFile outputGzip outputCache ipv6Enabled load_package_config() { config_load "$packageName" @@ -93,16 +106,63 @@ load_package_config() { config_get_bool debug "config" "debug" 0 config_get_bool allowNonAscii "config" "allow_non_ascii" 0 config_get_bool compressedCache "config" "compressed_cache" 0 + config_get_bool ipv6Enabled "config" "ipv6_enabled" 0 config_get bootDelay "config" "boot_delay" "120" config_get dlTimeout "config" "download_timeout" "20" + config_get curlRetry "config" "curl_retry" "3" config_get verbosity "config" "verbosity" "2" config_get led "config" "led" + config_get targetDNS "config" "dns" "dnsmasq.servers" config_get whitelist_domains "config" "whitelist_domain" config_get blacklist_domains "config" "blacklist_domain" config_get whitelist_domains_urls "config" "whitelist_domains_url" config_get blacklist_domains_urls "config" "blacklist_domains_url" config_get blacklist_hosts_urls "config" "blacklist_hosts_url" + if [ "$targetDNS" != "dnsmasq.addnhosts" ] && [ "$targetDNS" != "dnsmasq.conf" ] && \ + [ "$targetDNS" != "dnsmasq.servers" ] && [ "$targetDNS" != "unbound.adb_list" ]; then + targetDNS="dnsmasq.servers" + fi + + case "$targetDNS" in + dnsmasq.addnhosts) + outputFilter="$addnhostsOutputFilter" + outputFile="$addnhostsFile" + outputCache="$addnhostsCache" + outputGzip="$addnhostsGzip" + [ "$ipv6Enabled" -gt 0 ] && outputFilterIPv6="$addnhostsOutputFilterIPv6" + rm -f "$dnsmasqFile" "$dnsmasqCache" "$dnsmasqGzip" + rm -f "$serversFile" "$serversCache" "$serversGzip" + rm -f "$unboundFile" "$unboundCache" "$unboundGzip" + ;; + dnsmasq.conf) + outputFilter="$dnsmasqOutputFilter" + outputFile="$dnsmasqFile" + outputCache="$dnsmasqCache" + outputGzip="$dnsmasqGzip" + rm -f "$addnhostsFile" "$addnhostsCache" "$addnhostsGzip" + rm -f "$serversFile" "$serversCache" "$serversGzip" + rm -f "$unboundFile" "$unboundCache" "$unboundGzip" + ;; + dnsmasq.servers) + outputFilter="$serversOutputFilter" + outputFile="$serversFile" + outputCache="$serversCache" + outputGzip="$serversGzip" + rm -f "$dnsmasqFile" "$dnsmasqCache" "$dnsmasqGzip" + rm -f "$addnhostsFile" "$addnhostsCache" "$addnhostsGzip" + rm -f "$unboundFile" "$unboundCache" "$unboundGzip" + ;; + unbound.adb_list) + outputFilter="$unboundOutputFilter" + outputFile="$unboundFile" + outputCache="$unboundCache" + outputGzip="$unboundGzip" + rm -f "$dnsmasqFile" "$dnsmasqCache" "$dnsmasqGzip" + rm -f "$addnhostsFile" "$addnhostsCache" "$addnhostsGzip" + rm -f "$serversFile" "$serversCache" "$serversGzip" + ;; + esac if [ -z "${verbosity##*[!0-9]*}" ] || [ "$verbosity" -lt 0 ] || [ "$verbosity" -gt 2 ]; then verbosity=1 fi @@ -110,7 +170,7 @@ load_package_config() { . /usr/share/libubox/jshn.sh # Prefer curl because it supports the file: scheme. if [ -x /usr/bin/curl ] ; then - dl_command="curl --insecure --connect-timeout $dlTimeout --silent" + dl_command="curl --insecure --retry $curlRetry --connect-timeout $dlTimeout --silent" dl_flag="-o" else dl_command="wget --no-check-certificate --timeout $dlTimeout -q" @@ -129,15 +189,20 @@ is_enabled() { fi if [ "$serviceEnabled" -eq 0 ]; then - if [ "$1" = "on_start" ]; then - output "$packageName is currently disabled.\\n" - output "Run the following commands before starting service again:\\n" - output "uci set $packageName.config.enabled='1'; uci commit;\\n" - fi + case "$1" in + on_start) + output "$packageName is currently disabled.\\n" + output "Run the following commands before starting service again:\\n" + output "uci set ${packageName}.config.enabled='1'; uci commit $packageName;\\n" + ;; + esac return 1 fi - [ ! -d ${dnsmasqFile%/*} ] && mkdir -p ${dnsmasqFile%/*} - testCompressedCache && return 0 + + [ ! -d "${outputFile%/*}" ] && mkdir -p "${outputFile%/*}" + [ ! -d "${outputCache%/*}" ] && mkdir -p "${outputFile%/*}" + [ ! -d "${outputGzip%/*}" ] && mkdir -p "${outputFile%/*}" + cacheOps 'testGzip' && return 0 network_flush_cache; network_find_wan wan_if; network_get_gateway wan_gw "$wan_if"; [ -n "$wan_gw" ] && return 0 output "$_ERROR_: $serviceName failed to discover WAN gateway.\\n"; return 1; @@ -145,51 +210,97 @@ is_enabled() { dnsmasq_kill() { killall -q -HUP dnsmasq; } dnsmasq_restart() { /etc/init.d/dnsmasq restart >/dev/null 2>&1; } -reload_dnsmasq() { +unbound_restart() { /etc/init.d/unbound restart >/dev/null 2>&1; } +reload_resolver() { + local param output_text case $1 in on_start) - if [ -s "$dnsmasqFile" ]; then - output 3 "Restarting DNSMASQ " - tmpfs set message "restarting DNSMASQ" - if dnsmasq_restart; then - tmpfs set status "$statusSuccess" - led_on "$led" - output_okn - else - output_failn - tmpfs set status "$statusFail" - tmpfs add error "DNSMASQ restart error" - output "$_ERROR_: $serviceName failed to restart DNSMASQ!\\n" - return 1 - fi - else + if [ ! -s "$outputFile" ]; then tmpfs set status "$statusFail" - tmpfs add error "Failed to create $dnsmasqFile file." + tmpfs add error "Error: Failed to create $outputFile file." output "$_ERROR_: $serviceName failed to create its data file!\\n" return 1 fi - ;; - on_stop) - [ -f $dnsmasqFile ] && mv $dnsmasqFile $cacheFile - output 3 "Restarting dnsmasq " - if dnsmasq_restart; then - led_off "$led" + case "$targetDNS" in + dnsmasq.addnhosts) + uci -q del_list dhcp.@dnsmasq[0].addnhosts="$addnhostsFile" + uci add_list dhcp.@dnsmasq[0].addnhosts="$addnhostsFile" + if [ -n "$(uci changes dhcp)" ]; then + uci commit dhcp + param=dnsmasq_restart + output_text="Restarting DNSMASQ" + else + param=dnsmasq_kill + output_text="Reloading DNSMASQ" + fi + ;; + dnsmasq.conf) + param=dnsmasq_restart + output_text="Restarting DNSMASQ" + ;; + dnsmasq.servers) + if [ "$(uci -q get dhcp.@dnsmasq[0].serversfile)" != "$serversFile" ]; then + uci set dhcp.@dnsmasq[0].serversfile="$serversFile" + uci commit dhcp + param=dnsmasq_restart + output_text="Restarting DNSMASQ" + else + param=dnsmasq_kill + output_text="Reloading DNSMASQ" + fi + ;; + unbound.adb_list) + param=unbound_restart + output_text="Restarting Unbound" + ;; + esac + output 1 "$output_text " + output 2 "$output_text " + tmpfs set message "$output_text" + if eval "$param"; then + tmpfs set status "$statusSuccess" + led_on "$led" output_okn - output "$serviceName stopped.\\n" - tmpfs set status "$statusStopped" - tmpfs del message - tmpfs del error - return 0 - else - output_failn; + else + output_fail tmpfs set status "$statusFail" - tmpfs add error "DNSMASQ restart error on stop" - output "$_ERROR_: $serviceName failed to restart DNSMASQ on stop !\\n" + tmpfs add error "Error: $output_text error." + output "$_ERROR_: $serviceName $output_text error!\\n" return 1 fi ;; - quiet | *) - dnsmasq_restart && return 0 || return 1 + on_stop) + cacheOps 'create' + case "$targetDNS" in + dnsmasq.addnhosts | dnsmasq.servers) + if [ -n "$(uci changes dhcp)" ]; then + uci -q commit dhcp + param=dnsmasq_restart + else + param=dnsmasq_kill + fi + ;; + dnsmasq.conf) + param=dnsmasq_restart + ;; + unbound.adb_list) + param=unbound_restart + ;; + esac + eval "$param" + return $? + ;; + quiet) + case "$targetDNS" in + dnsmasq.addnhosts | dnsmasq.servers | dnsmasq.conf) + param=dnsmasq_restart + ;; + unbound.adb_list) + param=unbound_restart + ;; + esac + eval "$param" + return $? ;; esac } @@ -198,13 +309,13 @@ tmpfs(){ local action="$1" instance="$2" value="$3" local status message error stats local readReload readRestart curReload curRestart ret - if [ -s "$JsonFile" ]; then - status="$(jsonfilter -i $JsonFile -l1 -e "@['data']['status']")" - message="$(jsonfilter -i $JsonFile -l1 -e "@['data']['message']")" - error="$(jsonfilter -i $JsonFile -l1 -e "@['data']['error']")" - stats="$(jsonfilter -i $JsonFile -l1 -e "@['data']['stats']")" - readReload="$(jsonfilter -i $JsonFile -l1 -e "@['data']['reload']")" - readRestart="$(jsonfilter -i $JsonFile -l1 -e "@['data']['restart']")" + if [ -s "$jsonFile" ]; then + status="$(jsonfilter -i $jsonFile -l1 -e "@['data']['status']")" + message="$(jsonfilter -i $jsonFile -l1 -e "@['data']['message']")" + error="$(jsonfilter -i $jsonFile -l1 -e "@['data']['error']")" + stats="$(jsonfilter -i $jsonFile -l1 -e "@['data']['stats']")" + readReload="$(jsonfilter -i $jsonFile -l1 -e "@['data']['reload']")" + readRestart="$(jsonfilter -i $jsonFile -l1 -e "@['data']['restart']")" fi case "$action" in get) @@ -218,7 +329,7 @@ tmpfs(){ stats) echo "$stats"; return;; triggers) - curReload="$allowNonAscii $parallelDL $debug $dlTimeout $whitelist_domains $blacklist_domains $whitelist_domains_urls $blacklist_domains_urls $blacklist_hosts_urls" + curReload="$allowNonAscii $parallelDL $debug $dlTimeout $whitelist_domains $blacklist_domains $whitelist_domains_urls $blacklist_domains_urls $blacklist_hosts_urls $targetDNS" curRestart="$compressedCache $forceDNS $led" if [ "$curReload" != "$readReload" ]; then ret="download" @@ -266,7 +377,7 @@ tmpfs(){ stats) stats="$value";; triggers) - readReload="$allowNonAscii $parallelDL $debug $dlTimeout $whitelist_domains $blacklist_domains $whitelist_domains_urls $blacklist_domains_urls $blacklist_hosts_urls" + readReload="$allowNonAscii $parallelDL $debug $dlTimeout $whitelist_domains $blacklist_domains $whitelist_domains_urls $blacklist_domains_urls $blacklist_hosts_urls $targetDNS" readRestart="$compressedCache $forceDNS $led" ;; esac @@ -282,10 +393,50 @@ tmpfs(){ json_add_string reload "$readReload" json_add_string restart "$readRestart" json_close_object - json_dump > "$JsonFile" + json_dump > "$jsonFile" sync } +cacheOps(){ + local R_TMP + case "$1" in + create|backup) + [ -f "$outputFile" ] && mv "$outputFile" "$outputCache" >/dev/null 2>/dev/null + return $? + ;; + restore|use) + [ -f "$outputCache" ] && mv "$outputCache" "$outputFile" >/dev/null 2>/dev/null + return $? + ;; + test) + [ -s "$outputCache" ] + return $? + ;; + testGzip) + [ -s "$outputGzip" ] && gzip -t -c "$outputGzip" + return $? + ;; + createGzip) + R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)" + if gzip < "$outputFile" > "$R_TMP"; then + if mv "$R_TMP" "$outputGzip"; then + rm -f "$R_TMP" + return 0 + else + rm -f "$R_TMP" + return 1 + fi + else + return 1 + fi + ;; + expand|unpack|expandGzip|unpackGzip) + [ -s "$outputGzip" ] && gzip -dc < "$outputGzip" > "$outputCache" + return $? + ;; + esac +} + is_chaos_calmer() { ubus -S call system board | grep -q "Chaos Calmer"; } remove_fw3_redirect() { @@ -321,9 +472,9 @@ process_url() { if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then return 1; fi label="${1##*//}"; label="${label%%/*}"; if [ "$2" = "hosts" ]; then - label="Hosts: $label"; filter="$h_filter"; + label="Hosts: $label"; filter="$hostsFilter"; else - label="Domains: $label"; filter="$d_filter"; + label="Domains: $label"; filter="$domainsFilter"; fi if [ "$3" = "blocked" ]; then type="Blocked"; D_TMP="$B_TMP"; @@ -334,35 +485,41 @@ process_url() { R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)" done if ! $dl_command "$1" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then - output 2 "[DL] $type $label $__FAIL__\\n" output 1 "$_FAIL_" -# tmpfs add message "-" - dlStatus="${dlStatus}-" - tmpfs add error "Error downloading ${1}." - return 0 + output 2 "[DL] $type $label $__FAIL__\\n" + echo -e -n "Error: downloading '${1}'.\\n" >> "${jsonFile}.error" + else + sed -i "$filter" "$R_TMP" + if [ ! -s "$R_TMP" ]; then + output 1 "$_FAIL_" + output 2 "[DL] $type $label $__FAIL__\\n" + echo -e -n "Error: parsing '${1}'.\\n" >> "${jsonFile}.error" + else + cat "${R_TMP}" >> "$D_TMP" + output 1 "$_OK_" + output 2 "[DL] $type $label $__OK__\\n" + fi fi - sed -i "$filter" "$R_TMP" - cat "${R_TMP}" >> "$D_TMP" - rm -f "${R_TMP}" >/dev/null 2>/dev/null - output 2 "[DL] $type $label $__OK__\\n" - output 1 "$_OK_" -# tmpfs add message "+" - dlStatus="${dlStatus}+" + rm -f "$R_TMP" return 0 } download_lists() { - local i hf w_filter j=0 R_TMP + local hf w_filter j=0 R_TMP tmpfs set message "${statusDownloading}..." - for i in $A_TMP $B_TMP $cacheFile $dnsmasqFile; do [ -f $i ] && rm -f $i; done + rm -f "$A_TMP" "$B_TMP" "$outputFile" "$outputCache" "$outputGzip" if [ "$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo")" -lt 32 ]; then - output 3 "Low free memory, restarting dnsmasq..." - if reload_dnsmasq "quiet"; then output_okn; else output_failn; fi + output 3 "Low free memory, restarting resolver... " + if reload_resolver 'quiet'; then + output_okn + else + output_fail + fi fi touch $A_TMP; touch $B_TMP; output 1 "Downloading lists " - unset dlStatus + rm -f "${jsonFile}.error" if [ -n "$blacklist_hosts_urls" ]; then for hf in ${blacklist_hosts_urls}; do if [ "$parallelDL" -gt 0 ]; then @@ -391,27 +548,40 @@ download_lists() { done fi wait - dlStatus="${dlStatus//-/$xmark}" - dlStatus="${dlStatus//+/$checkmark}" - [ "$verbosity" = "1" ] && logmsg="${logmsg}${dlStatus}" + [ -s "${jsonFile}.error" ] && tmpfs add error "$(cat "${jsonFile}.error")" + rm -f "${jsonFile}.error" output 1 "\\n" - [ -n "$blacklist_domains" ] && for hf in ${blacklist_domains}; do echo "$hf" | sed "$d_filter" >> $B_TMP; done + [ -n "$blacklist_domains" ] && for hf in ${blacklist_domains}; do echo "$hf" | sed "$domainsFilter" >> $B_TMP; done whitelist_domains="${whitelist_domains} $(cat $A_TMP)" [ -n "$whitelist_domains" ] && for hf in ${whitelist_domains}; do hf=$(echo "$hf" | sed 's/\./\\./g'); w_filter="$w_filter/^${hf}$/d;/\\.${hf}$/d;"; done - if [ -s $B_TMP ]; then - output 1 "Processing downloads " - output 2 "Sorting combined list " - tmpfs set message "$statusProcessing: sorting combined list" - if sort $B_TMP | uniq > $A_TMP; then + + [ ! -s "$B_TMP" ] && return 1 + + output 1 "Processing downloads " + output 2 "Sorting combined list " + tmpfs set message "$statusProcessing: sorting combined list" + if [ "$allowNonAscii" -gt 0 ]; then + if sort "$B_TMP" | uniq > "$A_TMP"; then output_ok else - output_fail - tmpfs add error "Sorting error." + output_failn + tmpfs add error "Error: Sorting error." + fi + else + if sort "$B_TMP" | uniq | grep -E -v '[^a-zA-Z0-9=/.-]' > "$A_TMP"; then + output_ok + else + output_failn + tmpfs add error "Error: Sorting error." fi + fi -# TLD optimization written by Dirk Brenken (dev@brenken.org) + if [ "$targetDNS" == "dnsmasq.conf" ] || \ + [ "$targetDNS" == "dnsmasq.servers" ] || \ + [ "$targetDNS" == "unbound.adb_list" ]; then + # TLD optimization written by Dirk Brenken (dev@brenken.org) output 2 "Optimizing combined list " tmpfs set message "$statusProcessing: optimizing combined list" if awk -F "." '{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "$A_TMP" > "$B_TMP"; then @@ -421,99 +591,121 @@ $(cat $A_TMP)" if sort "$A_TMP" | uniq > "$B_TMP"; then output_ok else - output_fail - tmpfs add error "Data file optimization error." - mv $A_TMP $B_TMP + output_failn + tmpfs add error "Error: Data file optimization." + mv "$A_TMP" "$B_TMP" fi else - output_fail - tmpfs add error "Data file optimization error." + output_failn + tmpfs add error "Error: Data file optimization." fi else - output_fail - tmpfs add error "Data file optimization error." - mv $A_TMP $B_TMP + output_failn + tmpfs add error "Error: Data file optimization." + mv "$A_TMP" "$B_TMP" fi else - output_fail - tmpfs add error "Data file optimization error." + output_failn + tmpfs add error "Error: Data file optimization." fi else - output_fail - tmpfs add error "Data file optimization error." - mv $A_TMP $B_TMP + output_failn + tmpfs add error "Error: Data file optimization." + mv "$A_TMP" "$B_TMP" fi + else + mv "$A_TMP" "$B_TMP" + fi - output 2 "Whitelisting domains " - tmpfs set message "$statusProcessing: whitelisting domains" - if sed -i "$w_filter" $B_TMP; then - output_ok - else - output_fail - tmpfs add error "Whitelist processing error." - fi - output 2 "Formatting merged file " - tmpfs set message "$statusProcessing: formatting merged file" - if [ "$allowNonAscii" -gt 0 ]; then - if sed "$f_filter" $B_TMP > $A_TMP; then - output_ok - else - output_fail - tmpfs add error "Data file formatting error." - fi - else - if sed "$f_filter" $B_TMP | grep -E -v '[^a-zA-Z0-9=/.-]' > $A_TMP; then - output_ok - else - output_fail - tmpfs add error "Data file formatting error." - fi - fi + output 2 "Whitelisting domains " + tmpfs set message "$statusProcessing: whitelisting domains" + if sed -i "$w_filter" "$B_TMP"; then + output_ok + else + output_failn + tmpfs add error "Error: Whitelist processing." + fi - output 2 "Creating DNSMASQ config " - tmpfs set message "$statusProcessing: creating DNSMASQ blocklist" - if mv $A_TMP $dnsmasqFile; then + output 2 "Formatting merged file " + tmpfs set message "$statusProcessing: formatting merged file" + if [ -z "$outputFilterIPv6" ]; then + if sed "$outputFilter" "$B_TMP" > "$A_TMP"; then output_ok else - output_fail - tmpfs add error "Error moving data file ${A_TMP} to ${dnsmasqFile}." + output_failn + tmpfs add error "Error: Data file formatting." fi - - if [ "$compressedCache" -gt 0 ]; then - output 2 "Creating compressed cache " - tmpfs set message "$statusProcessing: creating compressed cache" - R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)" - if gzip < "$dnsmasqFile" > "$R_TMP"; then - mv "$R_TMP" "$compressedCacheFile" - output_ok - else - output_fail - rm -f "$R_TMP" - tmpfs add error "Error creating compressed cache." - fi + else + if sed "$outputFilter" "$B_TMP" > "$A_TMP" && \ + sed "$outputFilterIPv6" "$B_TMP" >> "$A_TMP"; then + output_ok else - rm -f "$compressedCacheFile" >/dev/null 2>&1 + output_failn + tmpfs add error "Error: Data file formatting." fi + fi - output 2 "Removing temporary files " - tmpfs set message "$statusProcessing: removing temporary files" - rm -f "/tmp/${packageName}_tmp.*" >/dev/null 2>&1 - for i in $A_TMP $B_TMP $cacheFile; do if [ -s $i ]; then rm -f $i || j=1; fi; done - if [ $j -eq 0 ]; then + case "$targetDNS" in + dnsmasq.addnhosts) + output 2 "Creating DNSMASQ addnhosts file " + tmpfs set message "$statusProcessing: creating DNSMASQ addnhosts file" + ;; + dnsmasq.conf) + output 2 "Creating DNSMASQ config file " + tmpfs set message "$statusProcessing: creating DNSMASQ config file" + ;; + dnsmasq.servers) + output 2 "Creating DNSMASQ servers file " + tmpfs set message "$statusProcessing: creating DNSMASQ servers file" + ;; + unbound.adb_list) + output 2 "Creating Unbound adb_list file " + tmpfs set message "$statusProcessing: creating Unbound adb_list file" + ;; + esac + if mv "$A_TMP" "$outputFile"; then + output_ok + else + output_failn + tmpfs add error "Error: moving data file '${A_TMP}' to '${outputFile}'." + fi + if [ "$compressedCache" -gt 0 ]; then + output 2 "Creating compressed cache " + tmpfs set message "$statusProcessing: creating compressed cache" + if cacheOps 'createGzip'; then output_ok else - output_fail - tmpfs add error "Error removing temporary files." + output_failn + tmpfs add error "Error: creating compressed cache." fi - output 1 "\\n" + else + rm -f "$outputGzip" + fi + output 2 "Removing temporary files " + tmpfs set message "$statusProcessing: removing temporary files" + rm -f "/tmp/${packageName}_tmp.*" "$A_TMP" "$B_TMP" "$outputCache" || j=1 + if [ $j -eq 0 ]; then + output_ok + else + output_failn + tmpfs add error "Error: removing temporary files." + fi + output 1 "\\n" +} + +boot() { + load_package_config + if create_lock; then + sleep "$bootDelay" + remove_lock + rc_procd start_service && rc_procd service_triggers fi } start_service() { - is_enabled "on_start" || return 1 - local ip status error action="$(tmpfs get triggers)" + is_enabled 'on_start' || return 1 + local ip status error action if create_lock; then - tmpfs set triggers procd_open_instance "main" procd_set_param command /bin/true procd_set_param stdout 1 @@ -544,6 +736,8 @@ start_service() { procd_close_instance status="$(tmpfs get status)" error="$(tmpfs get error)" + action="$(tmpfs get triggers)" + tmpfs set triggers tmpfs del status tmpfs del message tmpfs del error @@ -552,8 +746,10 @@ start_service() { case "$1" in download) action="download";; restart|*) - if [ ! -s "$dnsmasqFile" ] && [ ! -s "$cacheFile" ] && ! testCompressedCache; then + if [ ! -s "$outputFile" ] && ! cacheOps 'test' && ! cacheOps 'testGzip'; then action="download" + elif cacheOps 'test' || cacheOps 'testGzip'; then + action="start" fi if [ -n "$error" ]; then action="download" @@ -564,117 +760,135 @@ start_service() { case $action in download) - if [ -s "$dnsmasqFile" ] || [ -s "$cacheFile" ] || testCompressedCache; then - output "Force-reloading $serviceName...\\n" + if [ -s "$outputFile" ] || cacheOps 'test' || cacheOps 'testGzip'; then + output 0 "Force-reloading $serviceName... " + output 3 "Force-reloading $serviceName...\\n" tmpfs set status "$statusForceReloading" else - output "Starting $serviceName...\\n" + output 0 "Starting $serviceName... " + output 3 "Starting $serviceName...\\n" tmpfs set status "$statusStarting" fi download_lists - reload_dnsmasq "on_start" + reload_resolver 'on_start' ;; - restart|*) + restart|start) if [ "$action" == "restart" ]; then - output "Retarting $serviceName...\\n" + output 0 "Restarting $serviceName... " + output 3 "Restarting $serviceName...\\n" tmpfs set status "$statusRestarting" else - output "Starting $serviceName...\\n" + output 0 "Starting $serviceName... " + output 3 "Starting $serviceName...\\n" tmpfs set status "$statusStarting" fi - if testCompressedCache && [ ! -s "$cacheFile" ] && [ ! -s "$dnsmasqFile" ]; then + if cacheOps 'testGzip' && ! cacheOps 'test' && [ ! -s "$outputFile" ]; then output 3 "Found compressed cache file, unpacking it " tmpfs set message "found compressed cache file, unpacking it." - if gzip -dc < "${compressedCacheFile}" > "$cacheFile"; then + if cacheOps 'unpackGzip'; then output_okn else - output_failn + output_fail output "$_ERROR_: $serviceName failed to unpack compressed cache!\\n" - tmpfs add error "failed to unpack compressed cache!" + tmpfs add error "Error: Failed to unpack compressed cache." return 1 fi fi - if [ -s "$cacheFile" ] && [ ! -s "$dnsmasqFile" ]; then + if cacheOps 'test' && [ ! -s "$outputFile" ]; then output 3 "Found cache file, reusing it " tmpfs set message "found cache file, reusing it." - if mv "$cacheFile" "$dnsmasqFile"; then + if cacheOps 'restore'; then output_okn else - output_failn - tmpfs add error "Error moving $cacheFile to $dnsmasqFile." + output_fail + tmpfs add error "Error: moving '$outputCache' to '$outputFile'." fi fi - reload_dnsmasq "on_start" + reload_resolver 'on_start' ;; esac - if [ -s "$dnsmasqFile" ] && [ "$(tmpfs get status)" != "$statusFail" ]; then - output "$serviceName is blocking $(wc -l < $dnsmasqFile) domains "; output_okn; + if [ -s "$outputFile" ] && [ "$(tmpfs get status)" != "$statusFail" ]; then + output 0 "$__OK__\\n"; + local c="$(wc -l < "$outputFile")" + output 3 "$serviceName is blocking $c domains "; output_okn tmpfs del message - tmpfs set status "$statusSuccess: $(wc -l < $dnsmasqFile) domains blocked." - if [ -n "$(tmpfs get error)" ]; then - output "$_ERROR_: $(tmpfs get error)\\n" + tmpfs set status "$statusSuccess: $c domains blocked (with ${targetDNS})." + error="$(tmpfs get error)" + if [ -n "$error" ]; then + output "${error/Error:/$_ERROR_:}\\n" fi else + output 0 "$__FAIL__\\n"; tmpfs set status "$statusFail" - tmpfs add error "Failed to create DNSMASQ blocklist." + tmpfs add error "Error: Failed to create blocklist." fi remove_lock else - output "$serviceName: another instance is starting up "; output_failn; - return 1 + output 3 "$serviceName: another instance is starting up "; output_fail + return 0 fi } service_started() { procd_set_config_changed firewall; } service_stopped() { procd_set_config_changed firewall; } -restart_service() { rc_procd stop_service "restart"; rc_procd start_service "restart"; } +restart_service() { rc_procd start_service "restart"; } reload_service() { restart_service; } restart() { restart_service; } reload() { restart_service; } -dl() { rc_procd stop_service "restart"; rc_procd start_service "download"; } -killcache() { [ -s $cacheFile ] && rm -f $cacheFile >/dev/null 2>/dev/null; [ -s $compressedCacheFile ] && rm -f $compressedCacheFile >/dev/null 2>/dev/null; } +dl() { rc_procd start_service "download"; } + +killcache() { + rm -f "$addnhostsCache" "$addnhostsGzip" + rm -f "$dnsmasqCache" "$dnsmasqGzip" + rm -f "$serversCache" "$serversGzip" + rm -f "$unboundCache" "$unboundGzip" + return 0 +} + status() { - local status - if [ -n "$(tmpfs get status)" ]; then - status="$(tmpfs get status)" - fi - if [ -n "$status" ] && [ -n "$(tmpfs get message)" ]; then - status="${status}: $(tmpfs get message)" + local status="$(tmpfs get status)" error="$(tmpfs get error)" message="$(tmpfs get message)" + if [ -n "$status" ] && [ -n "$message" ]; then + status="${status}: $message" fi [ -n "$status" ] && output "$serviceName $status\\n" - [ -n "$(tmpfs get error)" ] && output "$_ERROR_: $(tmpfs get error)\\n" + [ -n "$error" ] && output "$error\\n" } stop_service() { load_package_config - if [ "$serviceEnabled" -gt 0 ]; then - if [ -z "$1" ] && [ -s "$dnsmasqFile" ]; then - [ -s "$dnsmasqFile" ] && mv "$dnsmasqFile" "$cacheFile" >/dev/null 2>/dev/null - output "Stopping $serviceName...\\n" - tmpfs del triggers - reload_dnsmasq "on_stop" - fi - else - [ -s "$dnsmasqFile" ] && mv "$dnsmasqFile" "$cacheFile" >/dev/null 2>/dev/null - reload_dnsmasq "quiet" - fi if is_chaos_calmer; then - fw3_setup "stop" + fw3_setup 'stop' + fi + if [ -s "$outputFile" ]; then + output "Stopping $serviceName... " + tmpfs del triggers + if reload_resolver 'on_stop'; then + led_off "$led" + output 0 "$__OK__\\n"; output_okn + tmpfs set status "$statusStopped" + tmpfs del message + else + output 0 "$__FAIL__\\n"; output_fail + tmpfs set status "$statusFail" + tmpfs add error "Error: error stopping $serviceName." + output "$_ERROR_: error stopping $serviceName!\\n" + fi fi } check() { load_package_config local string="$1" - if [ ! -f "$dnsmasqFile" ]; then - echo "No local blacklist ($dnsmasqFile) found." + if [ ! -f "$outputFile" ]; then + echo "No local blacklist ('$outputFile') found." elif [ -z "$string" ]; then echo "Usage: /etc/init.d/${packageName} check domain" - elif grep -m1 -q "$string" "$dnsmasqFile"; then - echo "Found $(grep -c "$string" "$dnsmasqFile") matches for $string in $dnsmasqFile:" - grep "$string" "$dnsmasqFile" | sed 's|local=/||;s|/$||;' + elif grep -m1 -q "$string" "$outputFile"; then + echo "Found $(grep -c "$string" "$outputFile") matches for '$string' in '$outputFile':" +# grep "$string" "$outputFile" | sed 's|local=/||;s|/$||;' + grep "$string" "$outputFile" else - echo "The $string is not found in current blacklist." + echo "The $string is not found in current blacklist ('$outputFile')." fi }