From be69e34ce3b4f4c144b4b9e7ec2a6b480ee4d44a Mon Sep 17 00:00:00 2001 From: Stan Grishin Date: Mon, 28 Aug 2023 21:13:13 +0000 Subject: [PATCH] adblock-fast: initial commit * this package replaces simple-adblock package * it was impossible to keep existing config structure and continue improving the simple-adblock the way I wanted, hence the new package name * the migration script for existing simple-adblock config is included in the uci-defaults file Signed-off-by: Stan Grishin --- net/adblock-fast/Makefile | 76 +++++ net/adblock-fast/files/README.md | 3 + .../files/adblock-fast.config.update} | 9 - .../files/etc/config/adblock-fast | 118 +++++++ .../files/etc/init.d/adblock-fast} | 311 ++++++++++-------- .../files/etc/uci-defaults/90-adblock-fast | 165 ++++++++++ net/{simple-adblock => adblock-fast}/test.sh | 0 net/simple-adblock/Makefile | 72 ---- net/simple-adblock/files/README.md | 3 - net/simple-adblock/files/simple-adblock.conf | 106 ------ .../files/simple-adblock.hotplug | 5 - 11 files changed, 532 insertions(+), 336 deletions(-) create mode 100644 net/adblock-fast/Makefile create mode 100644 net/adblock-fast/files/README.md rename net/{simple-adblock/files/simple-adblock.conf.update => adblock-fast/files/adblock-fast.config.update} (66%) create mode 100644 net/adblock-fast/files/etc/config/adblock-fast rename net/{simple-adblock/files/simple-adblock.init => adblock-fast/files/etc/init.d/adblock-fast} (90%) mode change 100644 => 100755 create mode 100644 net/adblock-fast/files/etc/uci-defaults/90-adblock-fast rename net/{simple-adblock => adblock-fast}/test.sh (100%) delete mode 100644 net/simple-adblock/Makefile delete mode 100644 net/simple-adblock/files/README.md delete mode 100644 net/simple-adblock/files/simple-adblock.conf delete mode 100644 net/simple-adblock/files/simple-adblock.hotplug diff --git a/net/adblock-fast/Makefile b/net/adblock-fast/Makefile new file mode 100644 index 0000000000..422b95c1ea --- /dev/null +++ b/net/adblock-fast/Makefile @@ -0,0 +1,76 @@ +# Copyright 2023 Stan Grishin (stangri@melmac.ca) +# TLD optimization written by Dirk Brenken (dev@brenken.org) +# This is free software, licensed under the GNU General Public License v3. + +include $(TOPDIR)/rules.mk + +PKG_NAME:=adblock-fast +PKG_VERSION:=1.0.0 +PKG_RELEASE:=1 +PKG_MAINTAINER:=Stan Grishin +PKG_LICENSE:=GPL-3.0-or-later + +include $(INCLUDE_DIR)/package.mk + +define Package/adblock-fast + SECTION:=net + CATEGORY:=Network + TITLE:=AdBlock Fast Service + URL:=https://docs.openwrt.melmac.net/adblock-fast/ + DEPENDS:=+jshn +curl + CONFLICTS:=simple-adblock + PROVIDES:=simple-adblock + PKGARCH:=all +endef + +define Package/adblock-fast/description +Fast AdBlocking script to block ad or abuse/malware domains with DNSMASQ or Unbound. +Script supports local/remote list of domains and hosts-files for both block-listing and allow-listing. +Please see https://docs.openwrt.melmac.net/adblock-fast/ for more information. +endef + +define Package/adblock-fast/conffiles +/etc/config/adblock-fast +endef + +define Build/Configure +endef + +define Build/Compile +endef + +define Package/adblock-fast/install + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/etc/init.d/adblock-fast $(1)/etc/init.d/adblock-fast + $(SED) "s|^\(readonly PKG_VERSION\).*|\1='$(PKG_VERSION)-$(PKG_RELEASE)'|" $(1)/etc/init.d/adblock-fast + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/etc/config/adblock-fast $(1)/etc/config/adblock-fast + $(INSTALL_DIR) $(1)/tmp + $(INSTALL_DATA) ./files/adblock-fast.config.update $(1)/tmp/adblock-fast.config.update + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/etc/uci-defaults/90-adblock-fast $(1)/etc/uci-defaults/90-adblock-fast +endef + +define Package/adblock-fast/postinst + #!/bin/sh + # check if we are on real system + if [ -z "$${IPKG_INSTROOT}" ]; then + sed -f /tmp/adblock-fast.config.update -i /etc/config/adblock-fast || true + /etc/init.d/adblock-fast enable + fi + exit 0 +endef + +define Package/adblock-fast/prerm + #!/bin/sh + # check if we are on real system + if [ -z "$${IPKG_INSTROOT}" ]; then + echo "Stopping service and removing rc.d symlink for adblock-fast" + /etc/init.d/adblock-fast stop || true + /etc/init.d/adblock-fast killcache || true + /etc/init.d/adblock-fast disable || true + fi + exit 0 +endef + +$(eval $(call BuildPackage,adblock-fast)) diff --git a/net/adblock-fast/files/README.md b/net/adblock-fast/files/README.md new file mode 100644 index 0000000000..827255ab2a --- /dev/null +++ b/net/adblock-fast/files/README.md @@ -0,0 +1,3 @@ +# README + +README has been moved to [https://docs.openwrt.melmac.net/adblock-fast/](https://docs.openwrt.melmac.net/adblock-fast/). diff --git a/net/simple-adblock/files/simple-adblock.conf.update b/net/adblock-fast/files/adblock-fast.config.update similarity index 66% rename from net/simple-adblock/files/simple-adblock.conf.update rename to net/adblock-fast/files/adblock-fast.config.update index dc2031734a..08f625eae1 100644 --- a/net/simple-adblock/files/simple-adblock.conf.update +++ b/net/adblock-fast/files/adblock-fast.config.update @@ -1,12 +1,3 @@ -s|whitelist_domains_url|allowed_domains_url|g -s|whitelist_domain|allowed_domain|g -s|blacklist_hosts_url|blocked_hosts_url|g -s|blacklist_domains_url|blocked_domains_url|g -s|blacklist_domain|blocked_domain|g -s|ssl.bblck.me|cdn.jsdelivr.net/gh/paulgb/BarbBlock|g -s|raw.githubusercontent.com/StevenBlack/hosts/|cdn.jsdelivr.net/gh/StevenBlack/hosts@|g -s|raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/|cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@|g -s|raw.githubusercontent.com/jawz101/MobileAdTrackers/|cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@|g s|http://winhelp2002.mvps.org/hosts.txt|https://winhelp2002.mvps.org/hosts.txt|g s|list blocked_domains_url 'https://dbl.oisd.nl/basic'|list blocked_adblockplus_url 'https://small.oisd.nl/'|g s|list blocked_domains_url 'https://dbl.oisd.nl/nsfw'|list blocked_adblockplus_url 'https://nsfw.oisd.nl/'|g diff --git a/net/adblock-fast/files/etc/config/adblock-fast b/net/adblock-fast/files/etc/config/adblock-fast new file mode 100644 index 0000000000..10cdf5186e --- /dev/null +++ b/net/adblock-fast/files/etc/config/adblock-fast @@ -0,0 +1,118 @@ +config adblock-fast 'config' + option enabled '0' + list allowed_domain 'cdn.jsdelivr.net' + option allow_non_ascii '0' + option canary_domains_icloud '0' + option canary_domains_mozilla '0' + option compressed_cache '0' + option compressed_cache_dir '/etc' + option config_update_enabled '0' + option config_update_url 'https://cdn.jsdelivr.net/gh/openwrt/packages/net/adblock-fast/files/adblock-fast.conf.update' + option curl_additional_param '' + option curl_max_file_size '30000000' + option curl_retry '3' + option debug '0' + option dns 'dnsmasq.servers' + option dnsmasq_instance '*' +# option dnsmasq_config_file_url 'https://big.oisd.nl/dnsmasq2' + option download_timeout '10' + option force_dns '1' + list force_dns_port '53' + list force_dns_port '853' +# ports listed below are used by some +# of the dnscrypt-proxy v1 resolvers +# list force_dns_port '553' +# list force_dns_port '1443' +# list force_dns_port '4343' +# list force_dns_port '4434' +# list force_dns_port '5443' +# list force_dns_port '8443' + option led 'none' + option parallel_downloads '1' + option procd_trigger_wan6 '0' + option procd_boot_wan_timeout '60' + option verbosity '2' + +config file_url + option url 'https://cdn.jsdelivr.net/gh/StevenBlack/hosts/hosts' + option size '6770929' + option action 'block' + option enabled '0' + +config file_url + option url 'https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_trackers_justdomains.txt' + option size '6241707' + option action 'block' + option enabled '0' + +config file_url + option url 'https://big.oisd.nl/' + option size '6163363' + option action 'block' + option enabled '0' + +config file_url + option url 'https://cdn.jsdelivr.net/gh/bongochong/CombinedPrivacyBlockLists/NoFormatting/cpbl-ctld.txt' + option size '2608152' + option action 'block' + option enabled '0' + +config file_url + option url 'http://sysctl.org/cameleon/hosts' + option size '638545' + option action 'block' + option enabled '0' + +config file_url + option url 'https://cdn.jsdelivr.net/gh/kboghdady/youTube_ads_4_pi-hole/black.list' + option size '553006' + option action 'block' + option enabled '0' + +config file_url + option url 'https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_clickthroughs_justdomains.txt' + option size '362170' + option action 'block' + option enabled '0' + +config file_url + option url 'https://someonewhocares.org/hosts/hosts' + option size '347410' + option action 'block' + option enabled '0' + +config file_url + option url 'https://winhelp2002.mvps.org/hosts.txt' + option size '334861' + option action 'block' + option enabled '0' + +config file_url + option url 'https://adaway.org/hosts.txt' + option size '243454' + option action 'block' + option enabled '0' + +config file_url + option url 'https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_ads_justdomains.txt' + option size '222595' + option action 'block' + option enabled '0' + +config file_url + option url 'https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_microsites_justdomains.txt' + option size '123275' + option action 'block' + option enabled '0' + +config file_url + option url 'https://pgl.yoyo.org/as/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext' + option size '99588' + option action 'block' + option enabled '0' + +config file_url + option url 'https://cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list/hosts.txt' + option size '11149' + option action 'block' + option enabled '0' diff --git a/net/simple-adblock/files/simple-adblock.init b/net/adblock-fast/files/etc/init.d/adblock-fast old mode 100644 new mode 100755 similarity index 90% rename from net/simple-adblock/files/simple-adblock.init rename to net/adblock-fast/files/etc/init.d/adblock-fast index ce51af0d79..f7fb5b3d23 --- a/net/simple-adblock/files/simple-adblock.init +++ b/net/adblock-fast/files/etc/init.d/adblock-fast @@ -1,6 +1,6 @@ #!/bin/sh /etc/rc.common -# Copyright 2017-2022 Stan Grishin (stangri@melmac.ca) -# shellcheck disable=SC1091,SC2016,SC3043,SC3057,SC3060 +# Copyright 2023 MOSSDeF, Stan Grishin (stangri@melmac.ca) +# shellcheck disable=SC1091,SC2015,SC2016,SC3037,SC3043,SC3045,SC3057,SC3060 # shellcheck disable=SC2034 START=94 @@ -28,7 +28,7 @@ else fi readonly PKG_VERSION='dev-test' -readonly packageName='simple-adblock' +readonly packageName='adblock-fast' readonly serviceName="$packageName $PKG_VERSION" readonly packageConfigFile="/etc/config/${packageName}" readonly dnsmasqAddnhostsFile="/var/run/${packageName}/dnsmasq.addnhosts" @@ -59,12 +59,13 @@ readonly unboundGzip="${packageName}.unbound.gz" readonly unboundFilter='s|^|local-zone: "|;s|$|" static|' readonly A_TMP="/var/${packageName}.hosts.a.tmp" readonly B_TMP="/var/${packageName}.hosts.b.tmp" -readonly jsonFile="/var/run/${packageName}/${packageName}.json" +readonly jsonFile="/dev/shm/$packageName-status.json" readonly sharedMemoryError="/dev/shm/$packageName-error" readonly hostsFilter='/localhost/d;/^#/d;/^[^0-9]/d;s/^0\.0\.0\.0.//;s/^127\.0\.0\.1.//;s/[[:space:]]*#.*$//;s/[[:cntrl:]]$//;s/[[:space:]]//g;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;' readonly domainsFilter='/^#/d;s/[[:space:]]*#.*$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;' readonly adBlockPlusFilter='/^#/d;/^!/d;s/[[:space:]]*#.*$//;s/^||//;s/\^$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;' readonly dnsmasqFileFilter='\|^server=/[[:alnum:]_.-].*/|!d' +readonly dnsmasq2FileFilter='\|^local=/[[:alnum:]_.-].*/|!d' readonly _OK_='\033[0;32m\xe2\x9c\x93\033[0m' readonly _FAIL_='\033[0;31m\xe2\x9c\x97\033[0m' readonly __OK__='\033[0;32m[\xe2\x9c\x93]\033[0m' @@ -88,6 +89,8 @@ outputGzip= outputCache= awk='awk' load_environment_flag= +allowed_url= +blocked_url= debug() { local i j; for i in "$@"; do eval "j=\$$i"; echo "${i}: ${j} "; done; } @@ -250,6 +253,36 @@ load_network() { output "${_ERROR_}: $(get_text 'errorNoWanGateway')!\\n"; return 1; } +append_url() { + local cfg="$1" var="$2" + local en action url + config_get en "$cfg" enabled '1' + config_get action "$cfg" action 'block' + config_get url "$cfg" url + if [ "$en" = '1' ]; then + if [ "$action" = 'allow' ]; then + allowed_url="${allowed_url:+$allowed_url }${url}" + else + blocked_url="${blocked_url:+$blocked_url }${url}" + fi + fi +} + +detect_file_type() { + local file="$1" + if [ "$(head -1 "$file")" = '[Adblock Plus]' ]; then + echo 'adBlockPlus' + elif grep -q '^server=' "$file"; then + echo 'dnsmasqFile' + elif grep -q '^local=' "$file"; then + echo 'dnsmasq2File' + elif grep -q '^0.0.0.0' "$file" || grep -q '^127.0.0.1' "$file"; then + echo 'hosts' + else + echo 'domains' + fi +} + load_environment() { local i j local validation_result="$1" param="$2" @@ -272,7 +305,7 @@ load_environment() { fi if [ "$debug" -ne 0 ]; then - exec 1>>/tmp/simple-adblock.log + exec 1>>"/tmp/$packageName.log" exec 2>&1 set -x fi @@ -444,20 +477,24 @@ load_environment() { fi # Prefer curl because it supports the file:// scheme. if is_present 'curl'; then - dl_command="curl --silent --insecure" + dl_command='curl --silent --insecure' dl_command="${dl_command}${curl_additional_param:+ $curl_additional_param}" dl_command="${dl_command}${curl_max_file_size:+ --max-filesize $curl_max_file_size}" dl_command="${dl_command}${curl_retry:+ --retry $curl_retry}" dl_command="${dl_command}${download_timeout:+ --connect-timeout $download_timeout}" - dl_flag="-o" + dl_flag='-o' elif is_present '/usr/libexec/wget-ssl'; then - dl_command="/usr/libexec/wget-ssl --no-check-certificate -q" + dl_command='/usr/libexec/wget-ssl --no-check-certificate -q' dl_command="${dl_command}${download_timeout:+ --timeout $download_timeout}" dl_flag="-O" + size_command='/usr/libexec/wget-ssl --no-check-certificate -q -O /dev/null --server-response' + size_command="${size_command}${download_timeout:+ --timeout $download_timeout}" elif is_present wget && wget --version 2>/dev/null | grep -q "+https"; then dl_command="wget --no-check-certificate -q" dl_command="${dl_command}${download_timeout:+ --timeout $download_timeout}" dl_flag="-O" + size_command='wget --no-check-certificate -q -O /dev/null --server-response' + size_command="${size_command}${download_timeout:+ --timeout $download_timeout}" else dl_command="uclient-fetch --no-check-certificate -q" dl_command="${dl_command}${download_timeout:+ --timeout $download_timeout}" @@ -473,6 +510,8 @@ load_environment() { else unset isSSLSupported fi + config_load "$packageName" + config_foreach append_url 'file_url' load_environment_flag=1 cache 'test' && return 0 cache 'test_gzip' && return 0 @@ -484,6 +523,26 @@ load_environment() { fi } +get_url_filesize() { + local url="$1" size size_command + [ -n "$url" ] || return 0 + is_present 'curl' || return 0 + size_command='curl --silent --insecure --fail --head --request GET' + size="$($size_command "$url" | grep -Po '^[cC]ontent-[lL]ength: \K\w+')" + echo -en "$size" +} + +get_local_filesize() { + local file="$1" size + [ -f "$file" ] || return 0 + if is_present stat; then + size="$(stat -c%s "$file")" + elif is_present wc; then + size="$(wc -c < "$file")" + fi + echo -en "$size" +} + resolver() { local cfg="$1" param="$2" case "$param" in @@ -520,10 +579,10 @@ dns() { fi config_load 'dhcp' - if [ "$dns_instance" = "*" ]; then + if [ "$dnsmasq_instance" = "*" ]; then config_foreach resolver 'dnsmasq' "$dns" - elif [ -n "$dns_instance" ]; then - for i in $dns_instance; do + elif [ -n "$dnsmasq_instance" ]; then + for i in $dnsmasq_instance; do resolver "@dnsmasq[$i]" "$dns" || resolver "$i" "$dns" done fi @@ -622,17 +681,19 @@ json() { json_load_file "$jsonFile" 2>/dev/null json_select 'data' 2>/dev/null for i in status message error stats reload restart; do - json_get_var $i "$i" 2>/dev/null + json_get_var "$i" "$i" 2>/dev/null done fi case "$action" in get) case "$param" in triggers) - curReload="$parallel_downloads $debug $download_timeout $allowed_domain $blocked_domain $allowed_domains_url \ - $blocked_adblockplus_url $blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \ - $dnsmasq_config_file_url $curl_additional_param $curl_max_file_size $curl_retry" - curRestart="$compressed_cache $compressed_cache_dir $force_dns $led $force_dns_port" + curReload="$parallel_downloads $debug $download_timeout \ + $allowed_domain $blocked_domain $allowed_url $blocked_url $dns \ + $config_update_enabled $config_update_url $dnsmasq_config_file_url \ + $curl_additional_param $curl_max_file_size $curl_retry" + curRestart="$compressed_cache $compressed_cache_dir $force_dns $led \ + $force_dns_port" if [ ! -s "$jsonFile" ]; then ret='on_boot' elif [ "$curReload" != "$reload" ]; then @@ -663,10 +724,12 @@ json() { set) case "$param" in triggers) - reload="$parallel_downloads $debug $download_timeout $allowed_domain $blocked_domain $allowed_domains_url \ - $blocked_adblockplus_url $blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \ - $dnsmasq_config_file_url $curl_additional_param $curl_max_file_size $curl_retry" - restart="$compressed_cache $compressed_cache_dir $force_dns $led $force_dns_port" + reload="$parallel_downloads $debug $download_timeout \ + $allowed_domain $blocked_domain $allowed_url $blocked_url $dns \ + $config_update_enabled $config_update_url $dnsmasq_config_file_url \ + $curl_additional_param $curl_max_file_size $curl_retry" + restart="$compressed_cache $compressed_cache_dir $force_dns $led \ + $force_dns_port" ;; *) eval "$param"='${value}${extras:+|$extras}';; @@ -729,24 +792,37 @@ cache() { esac } -process_url() { - local label type D_TMP R_TMP - if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then return 1; fi - label="${1##*//}"; label="${label%%/*}"; - case "$2" in - adbp) label="ADBPlus: $label"; filter="$adBlockPlusFilter" - ;; - dnsmasq) label="Dnsmasq: $label"; filter="$dnsmasqFileFilter" - ;; - domains) label="Domains: $label"; filter="$domainsFilter" - ;; - hosts) label="Hosts: $label"; filter="$hostsFilter" - ;; - esac - case "$3" in - allowed) type='Allowed'; D_TMP="$A_TMP" +_process_file_url() { + if [ "$2" != '0' ]; then + json add error "errorConfigValidationFail" + output "${_ERROR_}: $(get_text 'errorConfigValidationFail')!\\n" + output "Please check if the '$packageConfigFile' contains correct values for config options.\\n" + fi + if [ "$parallel_downloads" -gt 0 ]; then + process_file_url "$1" & + else + process_file_url "$1" + fi +} + +process_file_url() { + local cfg="$1" new_size + local label type D_TMP R_TMP filter + if [ -z "$cfg" ] || [ -n "${2}${3}" ]; then + url="$2" + action="$3" + fi + + [ "$enabled" = '1' ] || return 0 + [ -n "$url" ] || return 1 + + label="${url##*//}" + label="${label%%/*}" + label="File: $label" + case "$action" in + allow) type='Allowed'; D_TMP="$A_TMP" ;; - blocked) type='Blocked'; D_TMP="$B_TMP" + block) type='Blocked'; D_TMP="$B_TMP" ;; file) type='File'; D_TMP="$B_TMP" ;; @@ -760,20 +836,33 @@ process_url() { while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)" done - if ! $dl_command "$1" "$dl_flag" "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then + if [ -z "$url" ] || ! $dl_command "$url" "$dl_flag" "$R_TMP" 2>/dev/null || \ + [ ! -s "$R_TMP" ]; then output 1 "$_FAIL_" output 2 "[DL] $type $label $__FAIL__\\n" - echo "errorDownloadingList|${1}" >> "$sharedMemoryError" + echo "errorDownloadingList|${url}" >> "$sharedMemoryError" else - sed -i "$filter" "$R_TMP" + [ -n "$cfg" ] && new_size="$(get_local_filesize "$R_TMP")" + if [ -n "$new_size" ] && [ "$size" != "$new_size" ]; then + uci set "${packageName}.${cfg}.size=$size" + fi + format="$(detect_file_type "$R_TMP")" + case "$format" in + adBlockPlus) filter="$adBlockPlusFilter";; +# dnsmasqFile) filter="$dnsmasqFileFilter";; +# dnsmasq2File) filter="$dnsmasq2FileFilter";; + domains) filter="$domainsFilter";; + hosts) filter="$hostsFilter";; + esac + [ -n "$filter" ] && sed -i "$filter" "$R_TMP" if [ ! -s "$R_TMP" ]; then output 1 "$_FAIL_" - output 2 "[DL] $type $label $__FAIL__\\n" + output 2 "[DL] $type $label ($format) $__FAIL__\\n" echo "errorParsingList|${1}" >> "$sharedMemoryError" else cat "${R_TMP}" >> "$D_TMP" output 1 "$_OK_" - output 2 "[DL] $type $label $__OK__\\n" + output 2 "[DL] $type $label ($format) $__OK__\\n" fi fi rm -f "$R_TMP" @@ -798,15 +887,14 @@ download_dnsmasq_file() { touch $A_TMP; touch $B_TMP; output 1 'Downloading dnsmasq file ' rm -f "$sharedMemoryError" - process_url "$dnsmasq_config_file_url" 'dnsmasq' 'file' -# output 1 '\n' + process_file_url '' "$dnsmasq_config_file_url" 'file' if [ -s "$sharedMemoryError" ]; then while IFS= read -r line; do json add error "$line" done < "$sharedMemoryError" rm -f "$sharedMemoryError" fi - output 2 'Creating dnsmasq file ' + output 2 'Moving dnsmasq file ' if mv "$B_TMP" "$outputFile"; then output 2 "$__OK__\\n" else @@ -834,36 +922,15 @@ download_lists() { touch $A_TMP; touch $B_TMP; output 1 'Downloading lists ' rm -f "$sharedMemoryError" - for hf in ${blocked_hosts_url}; do - if [ "$parallel_downloads" -gt 0 ]; then - process_url "$hf" 'hosts' 'blocked' & - else - process_url "$hf" 'hosts' 'blocked' - fi - done - for hf in ${blocked_adblockplus_url}; do - if [ "$parallel_downloads" -gt 0 ]; then - process_url "$hf" 'adbp' 'blocked' & - else - process_url "$hf" 'adbp' 'blocked' - fi - done - for hf in ${blocked_domains_url}; do - if [ "$parallel_downloads" -gt 0 ]; then - process_url "$hf" 'domains' 'blocked' & - else - process_url "$hf" 'domains' 'blocked' - fi - done - for hf in ${allowed_domains_url}; do - if [ "$parallel_downloads" -gt 0 ]; then - process_url "$hf" 'domains' 'allowed' & - else - process_url "$hf" 'domains' 'allowed' - fi - done + config_load "$packageName" + config_foreach load_validate_file_url_section 'file_url' _process_file_url wait + if [ -n "$(uci changes "$packageName")" ]; then + output 2 "Saving updated file size(s) " + uci commit "$packageName" && output_okn || output_failn + fi output 1 '\n' + if [ -s "$sharedMemoryError" ]; then while IFS= read -r line; do json add error "$line" @@ -1207,65 +1274,26 @@ adb_config_update() { return 0 } +_config_add_url_size() { + local cfg="$1" url size + config_get url "$cfg" url + size="$(get_url_filesize "$url")" + output "$url${size:+: $size} " + if [ -n "$size" ]; then + uci set "${packageName}.${cfg}.size=$size" + output_okn + else + output_failn + fi +} + adb_sizes() { local i local validation_result="$3" load_environment "$validation_result" 'quiet' || return 1 - - echo "# $(date)" - for i in ${blocked_adblockplus_url}; do - [ "${i//melmac}" != "$i" ] && continue - if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then - echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')" - if is_greater "$(du -sk /tmp/sast)" "500"; then - echo "# block-list too big for most routers" - elif is_greater "$(du -sk /tmp/sast)" "100"; then - echo "# block-list may be too big for some routers" - fi - rm -rf /tmp/sast - echo " list blocked_adblockplus_url '$i'" - echo "" - else - echo "# site was down on last check" - echo "# list blocked_adblockplus_url '$i'" - echo "" - fi - done - for i in ${blocked_domains_url}; do - [ "${i//melmac}" != "$i" ] && continue - if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then - echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')" - if is_greater "$(du -sk /tmp/sast)" "500"; then - echo "# block-list too big for most routers" - elif is_greater "$(du -sk /tmp/sast)" "100"; then - echo "# block-list may be too big for some routers" - fi - rm -rf /tmp/sast - echo " list blocked_domains_url '$i'" - echo "" - else - echo "# site was down on last check" - echo "# list blocked_domains_url '$i'" - echo "" - fi - done - for i in ${blocked_hosts_url}; do - if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then - echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')" - if is_greater "$(du -sk /tmp/sast)" "500"; then - echo "# block-list too big for most routers" - elif is_greater "$(du -sk /tmp/sast)" "100"; then - echo "# block-list may be too big for some routers" - fi - rm -rf /tmp/sast - echo " list blocked_hosts_url '$i'" - echo "" - else - echo "# site was down on last check" - echo "# list blocked_hosts_url '$i'" - echo "" - fi - done + config_load "$packageName" + config_foreach _config_add_url_size 'file_url' + uci commit "$packageName" } # shellcheck disable=SC2120 @@ -1347,7 +1375,7 @@ adb_start() { output 3 "Starting $serviceName...\\n" json set status "statusStarting" fi - if [ -n "$dnsmasq_config_file_url" ]; then + if [ "$dns" = 'dnsmasq.conf' ] && [ -n "$dnsmasq_config_file_url" ]; then download_dnsmasq_file else download_lists @@ -1385,7 +1413,8 @@ adb_start() { procd_set_param stderr 1 procd_open_data json_add_string 'status' "$(json get status)" - json_add_string 'errors' "$(json get errors)" + json_add_string 'errors' "$(json get error)" + json_add_string 'warnings' "$(json get warning)" if [ -s "$outputFile" ]; then json_add_int 'entries' "$(wc -l < "$outputFile")" else @@ -1603,6 +1632,14 @@ stop_service() { load_validate_config 'config' adb_stop "'$*'"; } pause() { load_validate_config 'config' adb_pause "'$*'"; } version() { echo "$PKG_VERSION"; } +load_validate_file_url_section() { + uci_load_validate "$packageName" "$packageName" "$1" "$2" \ + 'enabled:bool:1' \ + 'action:or("allow", "block"):block' \ + 'size:or(uinteger, "")' \ + 'url:string' +} + load_validate_config() { . /lib/functions/network.sh . /usr/share/libubox/jshn.sh @@ -1629,13 +1666,9 @@ load_validate_config() { local procd_boot_wan_timeout local led local dns - local dns_instance + local dnsmasq_instance local allowed_domain - local allowed_domains_url - local blocked_adblockplus_url local blocked_domain - local blocked_domains_url - local blocked_hosts_url local dnsmasq_config_file_url uci_load_validate "$packageName" "$packageName" "$1" "${2}${3:+ $3}" \ 'enabled:bool:0' \ @@ -1650,7 +1683,7 @@ load_validate_config() { 'canary_domains_icloud:bool:0' \ 'canary_domains_mozilla:bool:0' \ 'config_update_enabled:bool:0' \ - 'config_update_url:string:https://cdn.jsdelivr.net/gh/openwrt/packages/net/simple-adblock/files/simple-adblock.conf.update' \ + 'config_update_url:string:https://cdn.jsdelivr.net/gh/openwrt/packages/net/adblock-fast/files/adblock-fast.conf.update' \ 'download_timeout:range(1,60):20' \ 'pause_timeout:range(10,120):60' \ 'curl_additional_param:or("", string)' \ @@ -1661,12 +1694,8 @@ load_validate_config() { 'procd_boot_wan_timeout:integer:60' \ 'led:or("", "none", file, device, string)' \ 'dns:or("dnsmasq.addnhosts", "dnsmasq.conf", "dnsmasq.ipset", "dnsmasq.nftset", "dnsmasq.servers", "unbound.adb_list"):dnsmasq.servers' \ - 'dns_instance:list(or(integer, string)):0' \ + 'dnsmasq_instance:list(or(integer, string)):*' \ 'allowed_domain:list(string)' \ - 'allowed_domains_url:list(string)' \ 'blocked_domain:list(string)' \ - 'blocked_adblockplus_url:list(string)' \ - 'blocked_domains_url:list(string)' \ - 'blocked_hosts_url:list(string)' \ 'dnsmasq_config_file_url:string' } diff --git a/net/adblock-fast/files/etc/uci-defaults/90-adblock-fast b/net/adblock-fast/files/etc/uci-defaults/90-adblock-fast new file mode 100644 index 0000000000..5366b46e64 --- /dev/null +++ b/net/adblock-fast/files/etc/uci-defaults/90-adblock-fast @@ -0,0 +1,165 @@ +#!/bin/sh +# Copyright 2023 MOSSDeF, Stan Grishin (stangri@melmac.ca) +# shellcheck disable=SC1091,SC2015,SC3037,SC3043,SC2317,SC3060 + +readonly packageName='adblock-fast' +readonly _OK_='\033[0;32m\xe2\x9c\x93\033[0m' +readonly _FAIL_='\033[0;31m\xe2\x9c\x97\033[0m' +readonly __OK__='\033[0;32m[\xe2\x9c\x93]\033[0m' +readonly __FAIL__='\033[0;31m[\xe2\x9c\x97]\033[0m' +readonly _ERROR_='\033[0;31mERROR\033[0m' +readonly _WARNING_='\033[0;33mWARNING\033[0m' +output() { +# Can take a single parameter (text) to be output at any verbosity +# Or target verbosity level and text to be output at specifc verbosity + local msg memmsg logmsg + local sharedMemoryOutput="/dev/shm/$packageName-output" + verbosity="${verbosity:-2}" + if [ "$#" -ne 1 ]; then + if [ $((verbosity & $1)) -gt 0 ] || [ "$verbosity" = "$1" ]; then + shift + else + return 0 + fi + fi + [ -t 1 ] && printf "%b" "$1" + msg="$1"; + if [ "$(printf "%b" "$msg" | wc -l)" -gt 0 ]; then + [ -s "$sharedMemoryOutput" ] && memmsg="$(cat "$sharedMemoryOutput")" + logmsg="$(printf "%b" "${memmsg}${msg}" | sed 's/\x1b\[[0-9;]*m//g')" + logger -t "${packageName:-service}" "$(printf "%b" "$logmsg")" + rm -f "$sharedMemoryOutput" + else + printf "%b" "$msg" >> "$sharedMemoryOutput" + fi +} +output_ok() { output 1 "$_OK_"; output 2 "$__OK__\\n"; } +output_okn() { output 1 "$_OK_\\n"; output 2 "$__OK__\\n"; } +output_fail() { output 1 "$_FAIL_"; output 2 "$__FAIL__\\n"; } +output_failn() { output 1 "$_FAIL_\\n"; output 2 "$__FAIL__\\n"; } +is_present() { command -v "$1" >/dev/null 2>&1; } +get_url_filesize() { + local url="$1" size size_command + [ -n "$1" ] || return 0 + is_present 'curl' || return 0 + size_command='curl --silent --insecure --fail --head --request GET' + size="$($size_command "$url" | grep -Po '^[cC]ontent-[lL]ength: \K\w+')" + echo -en "$size" +} + +# Transition from simple-adblock +_enable_url() { + local cfg="$1" url="$2" action="$3" + local u a + config_get u "$cfg" 'url' + config_get a "$cfg" 'action' 'block' + if [ "$u" = "$url" ] && [ "$a" = "$action" ]; then + uci del "${packageName}.${cfg}.enabled" && _found=1 + fi +} + +enable_add_url() { + local url="$1" action="$2" _found + config_load "$packageName" + config_foreach _enable_url 'file_url' "$url" "$action" + if [ -z "$_found" ]; then + uci add "${packageName}" 'file_url' >/dev/null 2>&1 + uci set "${packageName}.@file_url[-1].url=$url" + uci set "${packageName}.@file_url[-1].size=$(get_url_filesize "$url")" + uci set "${packageName}.@file_url[-1].action=$action" + fi +} + +if [ -s '/etc/config/simple-adblock' ] \ + && [ ! -s '/etc/config/adblock-fast-opkg' ] \ + && [ "$(uci get adblock-fast.config.enabled)" = '0' ]; then + cp -f '/etc/config/adblock-fast' '/etc/config/adblock-fast-opkg' + enabled="$(uci get simple-adblock.config.enabled)" + if [ -x '/etc/init.d/simple-adblock' ]; then + output "Stopping and disabling simple-adblock " + if /etc/init.d/simple-adblock stop >/dev/null 2>&1 \ + && /etc/init.d/simple-adblock disable \ + && uci set simple-adblock.config.enabled=0 \ + && uci commit simple-adblock; then + output_okn + else + output_failn + fi + else + output "Disabling simple-adblock." + if uci set simple-adblock.config.enabled=0 \ + && uci commit simple-adblock; then + output_okn + else + output_failn + fi + fi + output "Migrating simple-adblock config file " + for i in allow_non_ascii canary_domains_icloud canary_domains_mozilla \ + compressed_cache compressed_cache_dir config_update_enabled \ + curl_additional_param curl_max_file_size curl_retry download_timeout \ + debug dns dns_instance dnsmasq_config_file_url force_dns led \ + parallel_downloads procd_trigger_wan6 procd_boot_wan_timeout verbosity; do + j="$(uci -q get simple-adblock.config.${i})" + [ -n "$j" ] && uci set "${packageName}.config.${i}=${j}" + done + [ -n "$enabled" ] && uci set "${packageName}.config.enabled=${enabled}" + j="$(uci -q get simple-adblock.config.config_update_url)" + if [ "${j//simple-adblock/}" = "$j" ]; then + uci set "${packageName}.config.config_update_url=$j" + fi + ccd="$(uci get simple-adblock.config.compressed_cache_dir)" + ccd="${ccd:-/etc}" + for j in $(uci -q get simple-adblock.config.allowed_domain); do + [ -n "$j" ] && uci add_list "${packageName}.config.allowed_domain=${j}" + done + for j in $(uci -q get simple-adblock.config.blocked_domain); do + [ -n "$j" ] && uci add_list "${packageName}.config.blocked_domain=${j}" + done + for j in $(uci -q get simple-adblock.config.force_dns_port); do + [ -n "$j" ] && uci add_list "${packageName}.config.force_dns_port=${j}" + done + output_okn + + for i in allowed_domains_url blocked_adblockplus_url blocked_domains_url \ + blocked_hosts_url; do + output "Migrating simple-adblock ${i} " + for j in $(uci -q get simple-adblock.config.${i}); do + if [ "$i" = 'allowed_domains_url' ]; then + enable_add_url "$j" 'allow' + else + enable_add_url "$j" 'block' + fi + done + output_okn + done + uci commit "$packageName" + output "Migrating simple-adblock cache file(s) " + for i in '/var/run/simple-adblock/dnsmasq.addnhosts.cache' \ + '/var/run/simple-adblock/dnsmasq.conf.cache' \ + '/var/run/simple-adblock/dnsmasq.ipset.cache' \ + '/var/run/simple-adblock/dnsmasq.nftset.cache' \ + '/var/run/simple-adblock/dnsmasq.servers.cache' \ + '/var/run/simple-adblock/unbound.cache'; do + if [ -s "$i" ]; then + current_dir="$(dirname "$i")" + mkdir -p "${current_dir//simple-adblock/adblock-fast}" + mv -f "$i" "${i//simple-adblock/adblock-fast}" && output_okn || output_failn + fi + done + for i in 'simple-adblock.dnsmasq.addnhosts.gz' \ + 'simple-adblock.dnsmasq.conf.gz' \ + 'simple-adblock.dnsmasq.ipset.gz' \ + 'simple-adblock.dnsmasq.nftset.gz' \ + 'simple-adblock.dnsmasq.servers.gz' \ + 'simple-adblock.unbound.gz'; do + i="${ccd}/${i}" + if [ -s "$i" ]; then + mkdir -p "${ccd//simple-adblock/adblock-fast}" + mv -f "$i" "${i//simple-adblock/adblock-fast}" && output_okn || output_failn + fi + done + output_okn +fi + +exit 0 diff --git a/net/simple-adblock/test.sh b/net/adblock-fast/test.sh similarity index 100% rename from net/simple-adblock/test.sh rename to net/adblock-fast/test.sh diff --git a/net/simple-adblock/Makefile b/net/simple-adblock/Makefile deleted file mode 100644 index b3820ac818..0000000000 --- a/net/simple-adblock/Makefile +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2017-2022 Stan Grishin (stangri@melmac.ca) -# TLD optimization written by Dirk Brenken (dev@brenken.org) -# This is free software, licensed under the GNU General Public License v3. - -include $(TOPDIR)/rules.mk - -PKG_NAME:=simple-adblock -PKG_VERSION:=1.9.5 -PKG_RELEASE:=5 -PKG_MAINTAINER:=Stan Grishin -PKG_LICENSE:=GPL-3.0-or-later - -include $(INCLUDE_DIR)/package.mk - -define Package/simple-adblock - SECTION:=net - CATEGORY:=Network - TITLE:=Simple AdBlock Service - URL:=https://docs.openwrt.melmac.net/simple-adblock/ - DEPENDS:=+jshn +wget - PKGARCH:=all -endef - -define Package/simple-adblock/description -Simple adblock script to block ad or abuse/malware domains with DNSMASQ or Unbound. -Script supports local/remote list of domains and hosts-files for both block-listing and allow-listing. -Please see https://docs.openwrt.melmac.net/simple-adblock/ for more information. -endef - -define Package/simple-adblock/conffiles -/etc/config/simple-adblock -endef - -define Build/Configure -endef - -define Build/Compile -endef - -define Package/simple-adblock/install - $(INSTALL_DIR) $(1)/etc/init.d - $(INSTALL_DIR) $(1)/etc/config - $(INSTALL_DIR) $(1)/tmp - $(INSTALL_BIN) ./files/simple-adblock.init $(1)/etc/init.d/simple-adblock - $(SED) "s|^\(readonly PKG_VERSION\).*|\1='$(PKG_VERSION)-$(PKG_RELEASE)'|" $(1)/etc/init.d/simple-adblock - $(INSTALL_CONF) ./files/simple-adblock.conf $(1)/etc/config/simple-adblock - $(INSTALL_DATA) ./files/simple-adblock.conf.update $(1)/tmp/simple-adblock.conf.update -endef - -define Package/simple-adblock/postinst - #!/bin/sh - # check if we are on real system - if [ -z "$${IPKG_INSTROOT}" ]; then - sed -f /tmp/simple-adblock.conf.update -i /etc/config/simple-adblock || true - /etc/init.d/simple-adblock enable - fi - exit 0 -endef - -define Package/simple-adblock/prerm - #!/bin/sh - # check if we are on real system - if [ -z "$${IPKG_INSTROOT}" ]; then - echo "Stopping service and removing rc.d symlink for simple-adblock" - /etc/init.d/simple-adblock stop || true - /etc/init.d/simple-adblock killcache || true - /etc/init.d/simple-adblock disable || true - fi - exit 0 -endef - -$(eval $(call BuildPackage,simple-adblock)) diff --git a/net/simple-adblock/files/README.md b/net/simple-adblock/files/README.md deleted file mode 100644 index 09e19a4fc7..0000000000 --- a/net/simple-adblock/files/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# README - -README has been moved to [https://docs.openwrt.melmac.net/simple-adblock/](https://docs.openwrt.melmac.net/simple-adblock/). diff --git a/net/simple-adblock/files/simple-adblock.conf b/net/simple-adblock/files/simple-adblock.conf deleted file mode 100644 index e16c66bb0f..0000000000 --- a/net/simple-adblock/files/simple-adblock.conf +++ /dev/null @@ -1,106 +0,0 @@ -config simple-adblock 'config' - option enabled '0' - list allowed_domain 'cdn.jsdelivr.net' - option allow_non_ascii '0' - option canary_domains_icloud '0' - option canary_domains_mozilla '0' - option compressed_cache '0' - option compressed_cache_dir '/etc' - option config_update_enabled '0' - option config_update_url 'https://cdn.jsdelivr.net/gh/openwrt/packages/net/simple-adblock/files/simple-adblock.conf.update' - option curl_additional_param '' - option curl_max_file_size '30000000' - option curl_retry '3' - option download_timeout '10' - option debug '0' - option dns 'dnsmasq.servers' - option dns_instance '0' - option force_dns '1' - list force_dns_port '53' - list force_dns_port '853' -# ports listed below are used by some -# of the dnscrypt-proxy v1 resolvers -# list force_dns_port '553' -# list force_dns_port '1443' -# list force_dns_port '4343' -# list force_dns_port '4434' -# list force_dns_port '5443' -# list force_dns_port '8443' - option led 'none' - option parallel_downloads '1' - option procd_trigger_wan6 '0' - option procd_boot_wan_timeout '60' - option verbosity '2' - -# File size: 16.0K - list blocked_hosts_url 'https://adaway.org/hosts.txt' - -# File size: 20.0K - list blocked_hosts_url 'https://cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list/hosts.txt' - -# File size: 80.0K - list blocked_hosts_url 'https://pgl.yoyo.org/as/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext' - -# File size: 388.0K -# block-list may be too big for some routers -# block-list may block some video-streaming content -# list blocked_hosts_url 'https://cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers/hosts' - -# File size: 424.0K -# block-list may be too big for some routers - list blocked_hosts_url 'https://winhelp2002.mvps.org/hosts.txt' - -# File size: 432.0K -# block-list may be too big for some routers - list blocked_hosts_url 'https://someonewhocares.org/hosts/hosts' - -# File size: 624.0K -# block-list too big for most routers -# list blocked_hosts_url 'http://sysctl.org/cameleon/hosts' - -# File size: 1.4M -# block-list too big for most routers -# list blocked_adblockplus_url 'https://small.oisd.nl/' - -# File size: 1.6M -# block-list too big for most routers -# list blocked_hosts_url 'https://cdn.jsdelivr.net/gh/StevenBlack/hosts/hosts' - -# File size: 2.0M -# block-list too big for most routers -# list blocked_domains_url https://cdn.jsdelivr.net/gh/bongochong/CombinedPrivacyBlockLists/NoFormatting/cpbl-ctld.txt - -# File size: 3.1M -# block-list too big for most routers -# list blocked_hosts_url 'https://hostsfile.mine.nu/Hosts' - -# File size: 8.3M -# enabling this will disable processing of any other block/allow-lists -# option dnsmasq_config_file_url 'https://dnsmasq.oisd.nl/' - -# File size: 5.0M -# block-list may be too big for some routers -# list blocked_domains_url 'https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_trackers_justdomains.txt' - -# File size: 6.2M -# block-list too big for most routers -# list blocked_adblockplus_url 'https://nsfw.oisd.nl/' - -# File size: 6.2M -# block-list too big for most routers -# list blocked_adblockplus_url 'https://big.oisd.nl/' - -# File size: 1.5M -# block-list too big for most routers -# list blocked_domains_url 'https://small.oisd.nl/domains' - -# File size: 7.8M -# block-list too big for most routers -# list blocked_domains_url 'https://nsfw.oisd.nl/domains' - -# File size: 19.9M -# block-list too big for most routers -# list blocked_domains_url 'https://big.oisd.nl/domains' - -# site was down on last check -# list blocked_domains_url 'http://support.it-mate.co.uk/downloads/hosts.txt' diff --git a/net/simple-adblock/files/simple-adblock.hotplug b/net/simple-adblock/files/simple-adblock.hotplug deleted file mode 100644 index 6dc495d04d..0000000000 --- a/net/simple-adblock/files/simple-adblock.hotplug +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -if [ "$ACTION" = "ifup" ]; then - sleep 10 && /etc/init.d/simple-adblock start hotplug & -fi -- 2.30.2