mwan3: improve startup performance; version 2.9.0 13041/head
authorAaron Goodman <aaronjg@stanford.edu>
Sun, 26 Jul 2020 21:21:50 +0000 (17:21 -0400)
committerAaron Goodman <aaronjg@stanford.edu>
Tue, 18 Aug 2020 02:02:36 +0000 (22:02 -0400)
improve startup and runtime performance by

1) moving common startup procedures out of hotplug script when called
from mwan3 start
2) reducing calls to iptables to check status of rules
3) consolidating iptables updates and updating with iptables-restore
4) do not wait for kill if nothing was killed
5) running interface hotplug scripts in parallel
6) eliminate operations in hotplug script that check status on every
single interface unnecessarily
7) consolidate how mwan3track makes hotplug calls
8) do not restart mwan3track on connected events

This is a significant refactor, but should not result in any breaking
changes or require users to update their configurations.

version bump to 2.9.0

Signed-off-by: Aaron Goodman <aaronjg@stanford.edu>
net/mwan3/Makefile
net/mwan3/files/etc/hotplug.d/iface/15-mwan3
net/mwan3/files/etc/hotplug.d/iface/16-mwan3-user
net/mwan3/files/etc/init.d/mwan3
net/mwan3/files/lib/mwan3/common.sh
net/mwan3/files/lib/mwan3/mwan3.sh
net/mwan3/files/usr/sbin/mwan3
net/mwan3/files/usr/sbin/mwan3rtmon
net/mwan3/files/usr/sbin/mwan3track

index 6ae44ad357f6987259decf5fb2aa1ce30f67dbe6..65e2925c3b6da3688738ef4b35d5aac005d9c30c 100644 (file)
@@ -8,8 +8,8 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=mwan3
-PKG_VERSION:=2.8.12
-PKG_RELEASE:=2
+PKG_VERSION:=2.9.0
+PKG_RELEASE:=1
 PKG_MAINTAINER:=Florian Eckert <fe@dev.tdt.de>
 PKG_LICENSE:=GPL-2.0
 
index 6898df81933af263363836bd73dba5cc3029fa6a..6a7e95ac4b7604dad0327dceee51a3aa46445e61 100644 (file)
 . /lib/functions/network.sh
 . /lib/mwan3/mwan3.sh
 . /usr/share/libubox/jshn.sh
+. /lib/mwan3/common.sh
 
-[ "$ACTION" == "ifup" -o "$ACTION" == "ifdown" ] || exit 1
+SCRIPTNAME="mwan3-hotplug"
+[ "$ACTION" = "ifup" ] || [ "$ACTION" = "ifdown" ] ||  [ "$ACTION" = "connected" ] || [ "$ACTION" = "disconnected" ] || exit 1
 [ -n "$INTERFACE" ] || exit 2
-
-if [ "$ACTION" == "ifup" ]; then
-       [ -n "$DEVICE" ] || exit 3
+if ( [ "$ACTION" = "ifup" ] || [ "$ACTION" = "connected" ] ) && [ -z "$DEVICE" ]; then
+       LOG notice "$ACTION called on $INTERFACE with no device set"
+       exit 3
 fi
 
-mwan3_lock "$ACTION" "$INTERFACE"
+[ "$MWAN3_STARTUP" = 1 ] || mwan3_lock "$ACTION" "$INTERFACE"
+
 config_load mwan3
 config_get_bool enabled globals 'enabled' '0'
 [ "${enabled}" -gt 0 ] || {
-       mwan3_unlock "$ACTION" "$INTERFACE"
+       [ "$MWAN3_STARTUP" = 1 ] || mwan3_unlock "$ACTION" "$INTERFACE"
+       LOG notice "mwan3 hotplug on $INTERFACE not called because globally disabled"
        mwan3_flush_conntrack "$INTERFACE" "$ACTION"
        exit 0
 }
 
+$IPT4 -S mwan3_hook &>/dev/null || {
+       mwan3_unlock "$ACTION" "$INTERFACE"
+       LOG warn "hotplug called on $INTERFACE before mwan3 has been set up"
+       exit 0
+}
+
 mwan3_init
-mwan3_set_connected_iptables
-mwan3_set_custom_ipset
+[ "$MWAN3_STARTUP" = 1 ] || {
+       mwan3_set_connected_iptables
+       mwan3_set_custom_ipset
+}
+
+if [ "$MWAN3_STARTUP" != 1 ]; then
+       mwan3_set_user_iface_rules $INTERFACE $DEVICE
+fi
 
 config_get initial_state $INTERFACE initial_state "online"
 config_get_bool enabled $INTERFACE 'enabled' '0'
 [ "${enabled}" -eq 1 ] || {
-       mwan3_unlock "$ACTION" "$INTERFACE"
+       [ "$MWAN3_STARTUP" = 1 ] || mwan3_unlock "$ACTION" "$INTERFACE"
+       LOG notice "mwan3 hotplug on $INTERFACE not called because interface disabled"
        exit 0
 }
 
-if [ "$ACTION" = "ifup" ]; then
-       config_get family $INTERFACE family ipv4
-       if [ "$family" = "ipv4" ]; then
-               ubus call network.interface.${INTERFACE}_4 status &>/dev/null
-               if [ "$?" -eq "0" ]; then
-                       network_get_ipaddr src_ip ${INTERFACE}_4
-               else
-                       network_get_ipaddr src_ip ${INTERFACE}
-               fi
-               [ -n "$src_ip" ] || src_ip="0.0.0.0"
-       elif [ "$family" = "ipv6" ]; then
-               ubus call network.interface.${INTERFACE}_6 status &>/dev/null
-               if [ "$?" -eq "0" ]; then
-                       network_get_ipaddr6 src_ip ${INTERFACE}_6
-               else
-                       network_get_ipaddr6 src_ip ${INTERFACE}
-               fi
-               [ -n "$src_ip" ] || src_ip="::"
-       fi
-fi
+trackpid=$(pgrep -f "mwan3track $INTERFACE ")
 
 if [ "$initial_state" = "offline" ]; then
-       json_load "$(ubus call mwan3 status '{"section":"interfaces"}')"
-       json_select "interfaces"
-       json_select "${INTERFACE}"
-       json_get_var running running
-       json_get_var status status
+       status=$(cat $MWAN3TRACK_STATUS_DIR/$INTERFACE/STATUS 2>/dev/null || echo unknown)
 else
        status=online
-       running=1
 fi
 
-$LOG notice "Execute "$ACTION" event on interface $INTERFACE (${DEVICE:-unknown})"
+[ -z "$TRUE_INTERFACE" ] && mwan3_get_true_iface TRUE_INTERFACE $INTERFACE
+
+binary_status=$status
+[ "$binary_status" = "online" ] || binary_status=offline
+
+LOG notice "Execute "$ACTION" event on interface $INTERFACE (${DEVICE:-unknown})"
 
 case "$ACTION" in
-       ifup)
-               mwan3_set_general_rules
-               mwan3_set_general_iptables
+       ifup|connected)
                mwan3_create_iface_iptables $INTERFACE $DEVICE
                mwan3_create_iface_rules $INTERFACE $DEVICE
                mwan3_create_iface_route $INTERFACE $DEVICE
-               if [ "${running}" -eq 1 ] && [ "${status}" = "online" ]; then
-                       $LOG notice "Starting tracker on interface $INTERFACE (${DEVICE:-unknown})"
-                       mwan3_set_iface_hotplug_state $INTERFACE "online"
-                       mwan3_track $INTERFACE $DEVICE "online" "$src_ip"
+               [ "$MWAN3_STARTUP" != 1 ] && mwan3_add_non_default_iface_route $INTERFACE $DEVICE
+               mwan3_set_iface_hotplug_state $INTERFACE "$binary_status"
+
+               mwan3_get_src_ip src_ip "$TRUE_INTERFACE"
+               if [ -n "${trackpid}" ]; then
+                       device_pid=$(pgrep -f "mwan3track $INTERFACE $DEVICE ")
+                       if [ "$device_pid" = "$trackpid" ]; then
+                               [ "$ACTION" = ifup ] && kill -USR2 "$trackpid"
+                       else
+                               mwan3_track $INTERFACE $DEVICE "$binary_status" "$src_ip"
+                               LOG notice "Restarted tracker [$!] on interface $INTERFACE (${DEVICE:-unknown})"
+                       fi
                else
-                       $LOG notice "Starting tracker on interface $INTERFACE (${DEVICE:-unknown})"
-                       mwan3_set_iface_hotplug_state $INTERFACE "offline"
-                       mwan3_track $INTERFACE $DEVICE "offline" "$src_ip"
+                       mwan3_track $INTERFACE $DEVICE "$binary_status" "$src_ip"
+                       LOG notice "Started tracker [$!] on interface $INTERFACE (${DEVICE:-unknown})"
                fi
-               mwan3_set_policies_iptables
-               mwan3_set_user_rules
+               [ "$MWAN3_STARTUP" != 1 ] && [ "$binary_status" == "online" ] && mwan3_set_policies_iptables
+
        ;;
-       ifdown)
+       ifdown|disconnected)
                mwan3_set_iface_hotplug_state $INTERFACE "offline"
                mwan3_delete_iface_ipset_entries $INTERFACE
-               mwan3_track_signal $INTERFACE $DEVICE
+               mwan3_delete_iface_rules $INTERFACE
+               mwan3_delete_iface_route $INTERFACE
+               mwan3_delete_iface_iptables $INTERFACE
+               if [ "$ACTION" = "ifdown" ]; then
+                       [ -n "$trackpid" ] && kill -USR1 "$trackpid"
+               fi
                mwan3_set_policies_iptables
-               mwan3_set_user_rules
        ;;
 esac
-
-mwan3_unlock "$ACTION" "$INTERFACE"
-
+[ "$MWAN3_STARTUP" = 1 ] || mwan3_unlock "$ACTION" "$INTERFACE"
 exit 0
index af28b1f4f22c83244853812d36638a78b3ba9a66..2ec5c79a3c5e6e0c9ad418b467d087a4c25a0ec2 100644 (file)
@@ -4,22 +4,22 @@
        . /lib/functions.sh
        . /lib/mwan3/mwan3.sh
 
-       mwan3_lock "$ACTION" "user"
+       [ "$MWAN3_STARTUP" = 1 ] || mwan3_lock "$ACTION" "$DEVICE-user"
 
        config_load mwan3
        config_get_bool enabled globals 'enabled' '0'
        [ "${enabled}" -gt 0 ] || {
-               mwan3_unlock "$ACTION" "user"
+               [ "$MWAN3_STARTUP" = 1 ] || mwan3_unlock "$ACTION" "$DEVICE-user"
                exit 0
        }
 
        config_get_bool enabled "$INTERFACE" enabled 0
        [ "${enabled}" -eq 1 ] || {
-               mwan3_unlock "$ACTION" "user"
+               [ "$MWAN3_STARTUP" = 1 ] || mwan3_unlock "$ACTION" "$DEVICE-user"
                exit 0
        }
 
-       mwan3_unlock "$ACTION" "user"
+       [ "$MWAN3_STARTUP" = 1 ] || mwan3_unlock "$ACTION" "$DEVICE-user"
 
        env -i ACTION="$ACTION" INTERFACE="$INTERFACE" DEVICE="$DEVICE" \
                /bin/sh /etc/mwan3.user
index e0c65889aacce789591c757b2a8d489385670c65..ba9d920cbe9b8a279c03df3a800eed0589f90a1a 100755 (executable)
@@ -5,8 +5,9 @@ USE_PROCD=1
 
 boot() {
        . /lib/config/uci.sh
-       uci_toggle_state mwan3 globals enabled "1"
-       mwan3_boot=1
+       # disabled until mwan3 start runs so hotplug scripts
+       # do not start prematurely
+       uci_toggle_state mwan3 globals enabled "0"
        rc_procd start_service
 }
 
@@ -20,7 +21,6 @@ reload_service() {
 }
 
 start_service() {
-       [ -n "${mwan3_boot}" ] && return 0
        /usr/sbin/mwan3 start 1000>&-
 }
 
index 1af129919cf2e3af8c6717a451f940eb83c7e253..bb26327d588b8de63efa09730866d753af8e20bc 100644 (file)
@@ -4,3 +4,14 @@ get_uptime() {
        local uptime=$(cat /proc/uptime)
        echo "${uptime%%.*}"
 }
+
+SCRIPTNAME="$(basename "$0")"
+LOG()
+{
+       local facility=$1; shift
+       # in development, we want to show 'debug' level logs
+       # when this release is out of beta, the comment in the line below
+       # should be removed
+       [ "$facility" = "debug" ] && return
+       logger -t "$SCRIPTNAME[$$]" -p $facility "$*"
+}
index ad41030ad94947485713dc552d97d48241bb2c38..14af3ded2ae01366592816d7aa6d1f16c0073728 100644 (file)
@@ -1,4 +1,5 @@
 #!/bin/sh
+
 . /usr/share/libubox/jshn.sh
 
 IP4="ip -4"
@@ -6,7 +7,8 @@ IP6="ip -6"
 IPS="ipset"
 IPT4="iptables -t mangle -w"
 IPT6="ip6tables -t mangle -w"
-LOG="logger -t mwan3[$$] -p"
+IPT4R="iptables-restore -T mangle -w -n"
+IPT6R="ip6tables-restore -T mangle -w -n"
 CONNTRACK_FILE="/proc/net/nf_conntrack"
 IPv6_REGEX="([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|"
 IPv6_REGEX="${IPv6_REGEX}([0-9a-fA-F]{1,4}:){1,7}:|"
@@ -69,7 +71,7 @@ mwan3_update_dev_to_table()
 
 mwan3_update_iface_to_table()
 {
-       local _tid section family cfgtype curr_table _mwan3_iface_tbl
+       local _tid
        mwan3_iface_tbl=" "
        update_table()
        {
@@ -163,14 +165,14 @@ mwan3_init()
        else
                config_load mwan3
                config_get MMX_MASK globals mmx_mask '0x3F00'
-               echo "$MMX_MASK" > "${MWAN3_STATUS_DIR}/mmx_mask"
-               $LOG notice "Using firewall mask ${MMX_MASK}"
+               echo "$MMX_MASK"| tr 'A-F' 'a-f' > "${MWAN3_STATUS_DIR}/mmx_mask"
+               LOG debug "Using firewall mask ${MMX_MASK}"
 
                bitcnt=$(mwan3_count_one_bits MMX_MASK)
                mmdefault=$(((1<<bitcnt)-1))
                MWAN3_INTERFACE_MAX=$(($mmdefault-3))
                uci_toggle_state mwan3 globals iface_max "$MWAN3_INTERFACE_MAX"
-               $LOG notice "Max interface count is ${MWAN3_INTERFACE_MAX}"
+               LOG debug "Max interface count is ${MWAN3_INTERFACE_MAX}"
        fi
 
        # mark mask constants
@@ -187,14 +189,30 @@ mwan3_init()
 
 mwan3_lock() {
        lock /var/run/mwan3.lock
-#      $LOG debug "$1 $2 (lock)"
+       #LOG debug "$1 $2 (lock)"
 }
 
 mwan3_unlock() {
-#      $LOG debug "$1 $2 (unlock)"
+       #LOG debug "$1 $2 (unlock)"
        lock -u /var/run/mwan3.lock
 }
 
+mwan3_get_src_ip()
+{
+       local family _src_ip true_iface
+       true_iface=$2
+       unset "$1"
+       config_get family "$true_iface" family ipv4
+       if [ "$family" = "ipv4" ]; then
+               network_get_ipaddr _src_ip "$true_iface"
+               [ -n "$_src_ip" ] || _src_ip="0.0.0.0"
+       elif [ "$family" = "ipv6" ]; then
+               network_get_ipaddr6 _src_ip "$true_iface"
+               [ -n "$_src_ip" ] || _src_ip="::"
+       fi
+       export "$1=$_src_ip"
+}
+
 mwan3_get_iface_id()
 {
        local _tmp
@@ -202,6 +220,7 @@ mwan3_get_iface_id()
        _tmp="${mwan3_iface_tbl##* ${2}=}"
        _tmp=${_tmp%% *}
        export "$1=$_tmp"
+       new_val=$_tmp
 }
 
 mwan3_set_custom_ipset_v4()
@@ -209,8 +228,8 @@ mwan3_set_custom_ipset_v4()
        local custom_network_v4
 
        for custom_network_v4 in $($IP4 route list table "$1" | awk '{print $1}' | egrep '[0-9]{1,3}(\.[0-9]{1,3}){3}'); do
-               $LOG notice "Adding network $custom_network_v4 from table $1 to mwan3_custom_v4 ipset"
-               $IPS -! add mwan3_custom_v4_temp "$custom_network_v4"
+               LOG notice "Adding network $custom_network_v4 from table $1 to mwan3_custom_v4 ipset"
+               mwan3_push_update -! add mwan3_custom_v4 "$custom_network_v4"
        done
 }
 
@@ -219,47 +238,48 @@ mwan3_set_custom_ipset_v6()
        local custom_network_v6
 
        for custom_network_v6 in $($IP6 route list table "$1" | awk '{print $1}' | egrep "$IPv6_REGEX"); do
-               $LOG notice "Adding network $custom_network_v6 from table $1 to mwan3_custom_v6 ipset"
-               $IPS -! add mwan3_custom_v6_temp "$custom_network_v6"
+               LOG notice "Adding network $custom_network_v6 from table $1 to mwan3_custom_v6 ipset"
+               mwan3_push_update -! add mwan3_custom_v6 "$custom_network_v6"
        done
 }
 
 mwan3_set_custom_ipset()
 {
-       $IPS -! create mwan3_custom_v4 hash:net
-       $IPS create mwan3_custom_v4_temp hash:net
-       config_list_foreach "globals" "rt_table_lookup" mwan3_set_custom_ipset_v4
-       $IPS swap mwan3_custom_v4_temp mwan3_custom_v4
-       $IPS destroy mwan3_custom_v4_temp
+       local update=""
 
+       mwan3_push_update -! create mwan3_custom_v4 hash:net
+       config_list_foreach "globals" "rt_table_lookup" mwan3_set_custom_ipset_v4
 
-       $IPS -! create mwan3_custom_v6 hash:net family inet6
-       $IPS create mwan3_custom_v6_temp hash:net family inet6
+       mwan3_push_update -! create mwan3_custom_v6 hash:net family inet6
        config_list_foreach "globals" "rt_table_lookup" mwan3_set_custom_ipset_v6
-       $IPS swap mwan3_custom_v6_temp mwan3_custom_v6
-       $IPS destroy mwan3_custom_v6_temp
 
-       $IPS -! create mwan3_connected list:set
-       $IPS -! add mwan3_connected mwan3_custom_v4
-       $IPS -! add mwan3_connected mwan3_custom_v6
+       mwan3_push_update -! create mwan3_connected list:set
+       mwan3_push_update -! add mwan3_connected mwan3_custom_v4
+       mwan3_push_update -! add mwan3_connected mwan3_custom_v6
+       error=$(echo "$update" | $IPS restore 2>&1) || LOG error "set_custom_ipset: $error"
 }
 
 
 mwan3_set_connected_ipv4()
 {
        local connected_network_v4 candidate_list cidr_list
+       local ipv4regex='[0-9]{1,3}(\.[0-9]{1,3}){3}'
+       $IPS -! create mwan3_connected_v4 hash:net
+       $IPS create mwan3_connected_v4_temp hash:net
+
        candidate_list=""
        cidr_list=""
-       for connected_network_v4 in $($IP4 route | awk '{print $1}' | egrep '[0-9]{1,3}(\.[0-9]{1,3}){3}'); do
-               [ -z "${connected_network_v4##*/*}" ] &&
-                       cidr_list="$cidr_list $connected_network_v4" ||
-                               candidate_list="$candidate_list $connected_network_v4"
-       done
-
-       for connected_network_v4 in $($IP4 route list table 0 | awk '{print $2}' | egrep '[0-9]{1,3}(\.[0-9]{1,3}){3}'); do
-               [ -z "${connected_network_v4##*/*}" ] &&
-                       cidr_list="$cidr_list $connected_network_v4" ||
-                               candidate_list="$candidate_list $connected_network_v4"
+       route_lists()
+       {
+               $IP4 route | awk '{print $1}'
+               $IP4 route list table 0 | awk '{print $2}'
+       }
+       for connected_network_v4 in $(route_lists | egrep "$ipv4regex"); do
+               if [ -z "${connected_network_v4##*/*}" ]; then
+                       cidr_list="$cidr_list $connected_network_v4"
+               else
+                       candidate_list="$candidate_list $connected_network_v4"
+               fi
        done
 
        for connected_network_v4 in $cidr_list; do
@@ -279,41 +299,35 @@ mwan3_set_connected_ipv4()
 
 mwan3_set_connected_iptables()
 {
-       local connected_network_v6 source_network_v6
-
-       $IPS -! create mwan3_connected_v4 hash:net
-       $IPS create mwan3_connected_v4_temp hash:net
-
+       local connected_network_v6 source_network_v6 error
+       local update=""
        mwan3_set_connected_ipv4
 
        [ $NO_IPV6 -eq 0 ] && {
-               $IPS -! create mwan3_connected_v6 hash:net family inet6
-               $IPS create mwan3_connected_v6_temp hash:net family inet6
+               mwan3_push_update -! create mwan3_connected_v6 hash:net family inet6
+               mwan3_push_update flush mwan3_connected_v6
 
                for connected_network_v6 in $($IP6 route | awk '{print $1}' | egrep "$IPv6_REGEX"); do
-                       $IPS -! add mwan3_connected_v6_temp "$connected_network_v6"
+                       mwan3_push_update -! add mwan3_connected_v6 "$connected_network_v6"
                done
-               $IPS swap mwan3_connected_v6_temp mwan3_connected_v6
-               $IPS destroy mwan3_connected_v6_temp
 
-               $IPS -! create mwan3_source_v6 hash:net family inet6
-               $IPS create mwan3_source_v6_temp hash:net family inet6
+               mwan3_push_update -! create mwan3_source_v6 hash:net family inet6
                for source_network_v6 in $($IP6 addr ls | sed -ne 's/ *inet6 \([^ \/]*\).* scope global.*/\1/p'); do
-                       $IPS -! add mwan3_source_v6_temp "$source_network_v6"
+                       mwan3_push_update -! add mwan3_source_v6 "$source_network_v6"
                done
-               $IPS swap mwan3_source_v6_temp mwan3_source_v6
-               $IPS destroy mwan3_source_v6_temp
        }
 
-       $IPS -! create mwan3_connected list:set
-       $IPS -! add mwan3_connected mwan3_connected_v4
-       [ $NO_IPV6 -eq 0 ] && $IPS -! add mwan3_connected mwan3_connected_v6
+       mwan3_push_update -! create mwan3_connected list:set
+       mwan3_push_update flush mwan3_connected
+       mwan3_push_update -! add mwan3_connected mwan3_connected_v4
+       [ $NO_IPV6 -eq 0 ] && mwan3_push_update -! add mwan3_connected mwan3_connected_v6
 
-       $IPS -! create mwan3_dynamic_v4 hash:net
-       $IPS -! add mwan3_connected mwan3_dynamic_v4
+       mwan3_push_update -! create mwan3_dynamic_v4 hash:net
+       mwan3_push_update -! add mwan3_connected mwan3_dynamic_v4
 
-       [ $NO_IPV6 -eq 0 ] && $IPS -! create mwan3_dynamic_v6 hash:net family inet6
-       [ $NO_IPV6 -eq 0 ] && $IPS -! add mwan3_connected mwan3_dynamic_v6
+       [ $NO_IPV6 -eq 0 ] && mwan3_push_update -! create mwan3_dynamic_v6 hash:net family inet6
+       [ $NO_IPV6 -eq 0 ] && mwan3_push_update -! add mwan3_connected mwan3_dynamic_v6
+       error=$(echo "$update" | $IPS restore 2>&1) || LOG error "set_connected_iptables: $error"
 }
 
 mwan3_set_general_rules()
@@ -336,89 +350,96 @@ mwan3_set_general_rules()
 
 mwan3_set_general_iptables()
 {
-       local IPT
-
+       local IPT current update error
        for IPT in "$IPT4" "$IPT6"; do
                [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
-               if ! $IPT -S mwan3_ifaces_in &> /dev/null; then
-                       $IPT -N mwan3_ifaces_in
+               current="$($IPT -S)"
+               update="*mangle"
+               if [ -n "${current##*-N mwan3_ifaces_in*}" ]; then
+                       mwan3_push_update -N mwan3_ifaces_in
                fi
 
-               if ! $IPT -S mwan3_connected &> /dev/null; then
-                       $IPT -N mwan3_connected
+               if [ -n "${current##*-N mwan3_connected*}" ]; then
+                       mwan3_push_update -N mwan3_connected
                        $IPS -! create mwan3_connected list:set
-                       $IPT -A mwan3_connected \
-                               -m set --match-set mwan3_connected dst \
-                               -j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
+                       mwan3_push_update -A mwan3_connected \
+                                         -m set --match-set mwan3_connected dst \
+                                         -j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
                fi
 
-               if ! $IPT -S mwan3_rules &> /dev/null; then
-                       $IPT -N mwan3_rules
+               if [ -n "${current##*-N mwan3_rules*}" ]; then
+                       mwan3_push_update -N mwan3_rules
                fi
 
-               if ! $IPT -S mwan3_hook &> /dev/null; then
-                       $IPT -N mwan3_hook
+               if [ -n "${current##*-N mwan3_hook*}" ]; then
+                       mwan3_push_update -N mwan3_hook
                        # do not mangle ipv6 ra service
                        if [ "$IPT" = "$IPT6" ]; then
-                               $IPT6 -A mwan3_hook \
-                                       -p ipv6-icmp \
-                                       -m icmp6 --icmpv6-type 133 \
-                                       -j RETURN
-                               $IPT6 -A mwan3_hook \
-                                       -p ipv6-icmp \
-                                       -m icmp6 --icmpv6-type 134 \
-                                       -j RETURN
-                               $IPT6 -A mwan3_hook \
-                                       -p ipv6-icmp \
-                                       -m icmp6 --icmpv6-type 135 \
-                                       -j RETURN
-                               $IPT6 -A mwan3_hook \
-                                       -p ipv6-icmp \
-                                       -m icmp6 --icmpv6-type 136 \
-                                       -j RETURN
-                               $IPT6 -A mwan3_hook \
-                                       -p ipv6-icmp \
-                                       -m icmp6 --icmpv6-type 137 \
-                                       -j RETURN
+                               mwan3_push_update -A mwan3_hook \
+                                                 -p ipv6-icmp \
+                                                 -m icmp6 --icmpv6-type 133 \
+                                                 -j RETURN
+                               mwan3_push_update -A mwan3_hook \
+                                                 -p ipv6-icmp \
+                                                 -m icmp6 --icmpv6-type 134 \
+                                                 -j RETURN
+                               mwan3_push_update -A mwan3_hook \
+                                                 -p ipv6-icmp \
+                                                 -m icmp6 --icmpv6-type 135 \
+                                                 -j RETURN
+                               mwan3_push_update -A mwan3_hook \
+                                                 -p ipv6-icmp \
+                                                 -m icmp6 --icmpv6-type 136 \
+                                                 -j RETURN
+                               mwan3_push_update -A mwan3_hook \
+                                                 -p ipv6-icmp \
+                                                 -m icmp6 --icmpv6-type 137 \
+                                                 -j RETURN
                                # do not mangle outgoing echo request
-                               $IPT6 -A mwan3_hook \
-                                       -m set --match-set mwan3_source_v6 src \
-                                       -p ipv6-icmp \
-                                       -m icmp6 --icmpv6-type 128 \
-                                       -j RETURN
+                               mwan3_push_update -A mwan3_hook \
+                                                 -m set --match-set mwan3_source_v6 src \
+                                                 -p ipv6-icmp \
+                                                 -m icmp6 --icmpv6-type 128 \
+                                                 -j RETURN
 
                        fi
-                       $IPT -A mwan3_hook \
-                               -j CONNMARK --restore-mark --nfmask "$MMX_MASK" --ctmask "$MMX_MASK"
-                       $IPT -A mwan3_hook \
-                               -m mark --mark 0x0/$MMX_MASK \
-                               -j mwan3_ifaces_in
-                       $IPT -A mwan3_hook \
-                               -m mark --mark 0x0/$MMX_MASK \
-                               -j mwan3_connected
-                       $IPT -A mwan3_hook \
-                               -m mark --mark 0x0/$MMX_MASK \
-                               -j mwan3_rules
-                       $IPT -A mwan3_hook \
-                               -j CONNMARK --save-mark --nfmask "$MMX_MASK" --ctmask "$MMX_MASK"
-                       $IPT -A mwan3_hook \
-                               -m mark ! --mark $MMX_DEFAULT/$MMX_MASK \
-                               -j mwan3_connected
+                       mwan3_push_update -A mwan3_hook \
+                                         -j CONNMARK --restore-mark --nfmask "$MMX_MASK" --ctmask "$MMX_MASK"
+                       mwan3_push_update -A mwan3_hook \
+                                         -m mark --mark 0x0/$MMX_MASK \
+                                         -j mwan3_ifaces_in
+                       mwan3_push_update -A mwan3_hook \
+                                         -m mark --mark 0x0/$MMX_MASK \
+                                         -j mwan3_connected
+                       mwan3_push_update -A mwan3_hook \
+                                         -m mark --mark 0x0/$MMX_MASK \
+                                         -j mwan3_rules
+                       mwan3_push_update -A mwan3_hook \
+                                         -j CONNMARK --save-mark --nfmask "$MMX_MASK" --ctmask "$MMX_MASK"
+                       mwan3_push_update -A mwan3_hook \
+                                         -m mark ! --mark $MMX_DEFAULT/$MMX_MASK \
+                                         -j mwan3_connected
                fi
 
-               if ! $IPT -S PREROUTING | grep mwan3_hook &> /dev/null; then
-                       $IPT -A PREROUTING -j mwan3_hook
+               if [ -n "${current##*-A PREROUTING -j mwan3_hook*}" ]; then
+                       mwan3_push_update -A PREROUTING -j mwan3_hook
                fi
-
-               if ! $IPT -S OUTPUT | grep mwan3_hook &> /dev/null; then
-                       $IPT -A OUTPUT -j mwan3_hook
+               if [ -n "${current##*-A OUTPUT -j mwan3_hook*}" ]; then
+                       mwan3_push_update -A OUTPUT -j mwan3_hook
+               fi
+               mwan3_push_update COMMIT
+               mwan3_push_update ""
+               if [ "$IPT" = "$IPT4" ]; then
+                       error=$(echo "$update" | $IPT4R 2>&1) || LOG error "set_general_iptables: $error"
+               else
+                       error=$(echo "$update" | $IPT6R 2>&1) || LOG error "set_general_iptables: $error"
                fi
        done
 }
 
 mwan3_create_iface_iptables()
 {
-       local id family connected_name IPT
+       local id family connected_name IPT IPTR current update error
 
        config_get family "$1" family ipv4
        mwan3_get_iface_id id "$1"
@@ -428,43 +449,53 @@ mwan3_create_iface_iptables()
        if [ "$family" = "ipv4" ]; then
                connected_name=mwan3_connected
                IPT="$IPT4"
+               IPTR="$IPT4R"
                $IPS -! create $connected_name list:set
 
        elif [ "$family" = "ipv6" ] && [ $NO_IPV6 -eq 0 ]; then
                connected_name=mwan3_connected_v6
                IPT="$IPT6"
+               IPTR="$IPT6R"
                $IPS -! create $connected_name hash:net family inet6
        else
                return
        fi
-
-       if ! $IPT -S mwan3_ifaces_in &> /dev/null; then
-               $IPT -N mwan3_ifaces_in
+       current="$($IPT -S)"
+       update="*mangle"
+       if [ -n "${current##*-N mwan3_ifaces_in*}" ]; then
+               mwan3_push_update -N mwan3_ifaces_in
        fi
 
-       if ! $IPT -S "mwan3_iface_in_$1" &> /dev/null; then
-               $IPT -N "mwan3_iface_in_$1"
+       if  [ -n "${current##*-N mwan3_iface_in_$1*}" ]; then
+               mwan3_push_update -N "mwan3_iface_in_$1"
+       else
+               mwan3_push_update -F "mwan3_iface_in_$1"
        fi
 
-       $IPT -F "mwan3_iface_in_$1"
-       $IPT -A "mwan3_iface_in_$1" \
-            -i "$2" \
-            -m set --match-set $connected_name src \
-            -m mark --mark 0x0/$MMX_MASK \
-            -m comment --comment "default" \
-            -j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
-       $IPT -A "mwan3_iface_in_$1" \
-            -i "$2" \
-            -m mark --mark 0x0/$MMX_MASK \
-            -m comment --comment "$1" \
-            -j MARK --set-xmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK
+       mwan3_push_update -A "mwan3_iface_in_$1" \
+                         -i "$2" \
+                         -m set --match-set $connected_name src \
+                         -m mark --mark 0x0/$MMX_MASK \
+                         -m comment --comment "default" \
+                         -j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
+       mwan3_push_update -A "mwan3_iface_in_$1" \
+                         -i "$2" \
+                         -m mark --mark 0x0/$MMX_MASK \
+                         -m comment --comment "$1" \
+                         -j MARK --set-xmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK
+
+       if  [ -n "${current##*-A mwan3_ifaces_in -m mark --mark 0x0/$MMX_MASK -j mwan3_iface_in_${1}*}" ]; then
+               mwan3_push_update -A mwan3_ifaces_in \
+                                 -m mark --mark 0x0/$MMX_MASK \
+                                 -j "mwan3_iface_in_$1"
+               LOG debug "create_iface_iptables:  mwan3_iface_in_$1 not in iptables, adding"
+       else
+               LOG debug "create_iface_iptables:  mwan3_iface_in_$1 already in iptables, skip"
+       fi
 
-       $IPT -D mwan3_ifaces_in \
-            -m mark --mark 0x0/$MMX_MASK \
-            -j "mwan3_iface_in_$1" &> /dev/null
-       $IPT -A mwan3_ifaces_in \
-            -m mark --mark 0x0/$MMX_MASK \
-            -j "mwan3_iface_in_$1"
+       mwan3_push_update COMMIT
+       mwan3_push_update ""
+       error=$(echo "$update" | $IPTR 2>&1) || LOG error "create_iface_iptables: $error"
 
 }
 
@@ -568,7 +599,6 @@ mwan3_add_all_nondefault_routes()
        add_route()
        {
                let tid++
-               config_get family "$section" family ipv4
                [ -n "${active_tbls##* $tid *}" ] && return
                $IP route add table $tid $route_line ||
                        LOG warn "failed to add $route_line to table $tid"
@@ -615,7 +645,7 @@ mwan3_delete_iface_route()
 
 mwan3_create_iface_rules()
 {
-       local id family
+       local id family IP
 
        config_get family "$1" family ipv4
        mwan3_get_iface_id id "$1"
@@ -623,32 +653,23 @@ mwan3_create_iface_rules()
        [ -n "$id" ] || return 0
 
        if [ "$family" = "ipv4" ]; then
-
-               while [ -n "$($IP4 rule list | awk '$1 == "'$(($id+1000)):'"')" ]; do
-                       $IP4 rule del pref $(($id+1000))
-               done
-
-               while [ -n "$($IP4 rule list | awk '$1 == "'$(($id+2000)):'"')" ]; do
-                       $IP4 rule del pref $(($id+2000))
-               done
-
-               $IP4 rule add pref $(($id+1000)) iif "$2" lookup "$id"
-               $IP4 rule add pref $(($id+2000)) fwmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK lookup "$id"
+               IP="$IP4"
+       elif [ "$family" = "ipv6" ] && [ $NO_IPV6 -eq 0 ]; then
+               IP="$IP6"
+       else
+               return
        fi
 
-       if [ "$family" = "ipv6" ] && [ $NO_IPV6 -eq 0 ]; then
-
-               while [ -n "$($IP6 rule list | awk '$1 == "'$(($id+1000)):'"')" ]; do
-                       $IP6 rule del pref $(($id+1000))
-               done
+       while [ -n "$($IP rule list | awk '$1 == "'$(($id+1000)):'"')" ]; do
+               $IP rule del pref $(($id+1000))
+       done
 
-               while [ -n "$($IP6 rule list | awk '$1 == "'$(($id+2000)):'"')" ]; do
-                       $IP6 rule del pref $(($id+2000))
-               done
+       while [ -n "$($IP rule list | awk '$1 == "'$(($id+2000)):'"')" ]; do
+               $IP rule del pref $(($id+2000))
+       done
 
-               $IP6 rule add pref $(($id+1000)) iif "$2" lookup "$id"
-               $IP6 rule add pref $(($id+2000)) fwmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK lookup "$id"
-       fi
+       $IP rule add pref $(($id+1000)) iif "$2" lookup "$id"
+       $IP rule add pref $(($id+2000)) fwmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK lookup "$id"
 }
 
 mwan3_delete_iface_rules()
@@ -661,26 +682,20 @@ mwan3_delete_iface_rules()
        [ -n "$id" ] || return 0
 
        if [ "$family" = "ipv4" ]; then
-
-               while [ -n "$($IP4 rule list | awk '$1 == "'$(($id+1000)):'"')" ]; do
-                       $IP4 rule del pref $(($id+1000))
-               done
-
-               while [ -n "$($IP4 rule list | awk '$1 == "'$(($id+2000)):'"')" ]; do
-                       $IP4 rule del pref $(($id+2000))
-               done
+               IP="$IP4"
+       elif [ "$family" = "ipv6" ] && [ $NO_IPV6 -eq 0 ]; then
+               IP="$IP6"
+       else
+               return
        fi
 
-       if [ "$family" = "ipv6" ] && [ $NO_IPV6 -eq 0 ]; then
-
-               while [ -n "$($IP6 rule list | awk '$1 == "'$(($id+1000)):'"')" ]; do
-                       $IP6 rule del pref $(($id+1000))
-               done
+       while [ -n "$($IP rule list | awk '$1 == "'$(($id+1000)):'"')" ]; do
+               $IP rule del pref $(($id+1000))
+       done
 
-               while [ -n "$($IP6 rule list | awk '$1 == "'$(($id+2000)):'"')" ]; do
-                       $IP6 rule del pref $(($id+2000))
-               done
-       fi
+       while [ -n "$($IP rule list | awk '$1 == "'$(($id+2000)):'"')" ]; do
+               $IP rule del pref $(($id+2000))
+       done
 }
 
 mwan3_delete_iface_ipset_entries()
@@ -692,7 +707,7 @@ mwan3_delete_iface_ipset_entries()
        [ -n "$id" ] || return 0
 
        for setname in $(ipset -n list | grep ^mwan3_sticky_); do
-               for entry in $(ipset list "$setname" | grep "$(echo $(mwan3_id2mask id MMX_MASK) | awk '{ printf "0x%08x", $1; }')" | cut -d ' ' -f 1); do
+               for entry in $(ipset list "$setname" | grep "$(mwan3_id2mask id MMX_MASK | awk '{ printf "0x%08x", $1; }')" | cut -d ' ' -f 1); do
                        $IPS del "$setname" $entry
                done
        done
@@ -712,7 +727,7 @@ mwan3_rtmon()
 
 mwan3_track()
 {
-       local track_ip track_ips pid
+       local track_ips pids
 
        mwan3_list_track_ips()
        {
@@ -720,28 +735,21 @@ mwan3_track()
        }
        config_list_foreach "$1" track_ip mwan3_list_track_ips
 
-       kill -TERM $(pgrep -f "mwan3track $1 $2") > /dev/null 2>&1
-       sleep 1
-       kill -KILL $(pgrep -f "mwan3track $1 $2") > /dev/null 2>&1
+       # don't match device in case it changed from last launch
+       if pids=$(pgrep -f "mwan3track $1 "); then
+               kill -TERM $pids > /dev/null 2>&1
+               sleep 1
+               kill -KILL $(pgrep -f "mwan3track $1 ") > /dev/null 2>&1
+       fi
 
        if [ -n "$track_ips" ]; then
-               [ -x /usr/sbin/mwan3track ] && /usr/sbin/mwan3track "$1" "$2" "$3" "$4" $track_ips &
+               [ -x /usr/sbin/mwan3track ] && MWAN3_STARTUP=0 /usr/sbin/mwan3track "$1" "$2" "$3" "$4" $track_ips &
        fi
 }
 
-mwan3_track_signal()
-{
-       local pid
-
-       pid="$(pgrep -f "mwan3track $1 $2")"
-       [ "${pid}" != "" ] && {
-               kill -USR1 "${pid}"
-       }
-}
-
 mwan3_set_policy()
 {
-       local iface_count id iface family metric probability weight device is_lowest is_offline IPT total_weight
+       local id iface family metric probability weight device is_lowest is_offline IPT IPTR total_weight current update error
 
        is_lowest=0
        config_get iface "$1" interface
@@ -750,22 +758,26 @@ mwan3_set_policy()
 
        [ -n "$iface" ] || return 0
        network_get_device device "$iface"
-       [ "$metric" -gt $DEFAULT_LOWEST_METRIC ] && $LOG warn "Member interface $iface has >$DEFAULT_LOWEST_METRIC metric. Not appending to policy" && return 0
+       [ "$metric" -gt $DEFAULT_LOWEST_METRIC ] && LOG warn "Member interface $iface has >$DEFAULT_LOWEST_METRIC metric. Not appending to policy" && return 0
 
        mwan3_get_iface_id id "$iface"
 
+       [ -n "$id" ] || return 0
+
        [ "$(mwan3_get_iface_hotplug_state "$iface")" = "online" ]
        is_offline=$?
 
-       [ -n "$id" ] || return 0
-
        config_get family "$iface" family ipv4
 
        if [ "$family" = "ipv4" ]; then
                IPT="$IPT4"
+               IPTR="$IPT4R"
        elif [ "$family" = "ipv6" ]; then
                IPT="$IPT6"
+               IPTR="$IPT6R"
        fi
+       current="$($IPT -S)"
+       update="*mangle"
 
        if [ "$family" = "ipv4" ] && [ $is_offline -eq 0 ]; then
                if [ "$metric" -lt "$lowest_metric_v4" ]; then
@@ -791,11 +803,11 @@ mwan3_set_policy()
                fi
        fi
        if [ $is_lowest -eq 1 ]; then
-               $IPT -F "mwan3_policy_$policy"
-               $IPT -A "mwan3_policy_$policy" \
-                    -m mark --mark 0x0/$MMX_MASK \
-                    -m comment --comment "$iface $weight $weight" \
-                    -j MARK --set-xmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK
+               mwan3_push_update -F "mwan3_policy_$policy"
+               mwan3_push_update -A "mwan3_policy_$policy" \
+                                 -m mark --mark 0x0/$MMX_MASK \
+                                 -m comment --comment \"$iface $weight $weight\" \
+                                 -j MARK --set-xmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK
        elif [ $is_offline -eq 0 ]; then
                probability=$(($weight*1000/$total_weight))
                if [ "$probability" -lt 10 ]; then
@@ -808,63 +820,76 @@ mwan3_set_policy()
                        probability="1"
                fi
 
-               $IPT -I "mwan3_policy_$policy" \
-                       -m mark --mark 0x0/$MMX_MASK \
-                       -m statistic \
-                       --mode random \
-                       --probability "$probability" \
-                       -m comment --comment "$iface $weight $total_weight" \
-                       -j MARK --set-xmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK
+               mwan3_push_update -I "mwan3_policy_$policy" \
+                                 -m mark --mark 0x0/$MMX_MASK \
+                                 -m statistic \
+                                 --mode random \
+                                 --probability "$probability" \
+                                 -m comment --comment \"$iface $weight $total_weight\" \
+                                 -j MARK --set-xmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK
        elif [ -n "$device" ]; then
-               $IPT -S "mwan3_policy_$policy" | grep -q '.*--comment ".* [0-9]* [0-9]*"' || \
-                       $IPT -I "mwan3_policy_$policy" \
-                               -o "$device" \
-                               -m mark --mark 0x0/$MMX_MASK \
-                               -m comment --comment "out $iface $device" \
-                               -j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
+               echo "$current" | grep -q "^-A mwan3_policy_$policy.*--comment .* [0-9]* [0-9]*" ||
+                       mwan3_push_update -I "mwan3_policy_$policy" \
+                                         -o "$device" \
+                                         -m mark --mark 0x0/$MMX_MASK \
+                                         -m comment --comment \"out $iface $device\" \
+                                         -j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
        fi
+       mwan3_push_update COMMIT
+       mwan3_push_update ""
+       error=$(echo "$update" | $IPTR 2>&1) || LOG error "set_policy ($1): $error"
+
 }
 
 mwan3_create_policies_iptables()
 {
-       local last_resort lowest_metric_v4 lowest_metric_v6 total_weight_v4 total_weight_v6 policy IPT
+       local last_resort lowest_metric_v4 lowest_metric_v6 total_weight_v4 total_weight_v6 policy IPT current update error
 
        policy="$1"
 
        config_get last_resort "$1" last_resort unreachable
 
        if [ "$1" != "$(echo "$1" | cut -c1-15)" ]; then
-               $LOG warn "Policy $1 exceeds max of 15 chars. Not setting policy" && return 0
+               LOG warn "Policy $1 exceeds max of 15 chars. Not setting policy" && return 0
        fi
 
        for IPT in "$IPT4" "$IPT6"; do
                [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
-               if ! $IPT -S "mwan3_policy_$1" &> /dev/null; then
-                       $IPT -N "mwan3_policy_$1"
+               current="$($IPT -S)"
+               update="*mangle"
+               if [ -n "${current##*-N mwan3_policy_$1*}" ]; then
+                       mwan3_push_update  -N "mwan3_policy_$1"
                fi
 
-               $IPT -F "mwan3_policy_$1"
+               mwan3_push_update -F "mwan3_policy_$1"
 
                case "$last_resort" in
                        blackhole)
-                               $IPT -A "mwan3_policy_$1" \
-                                       -m mark --mark 0x0/$MMX_MASK \
-                                       -m comment --comment "blackhole" \
-                                       -j MARK --set-xmark $MMX_BLACKHOLE/$MMX_MASK
-                       ;;
+                               mwan3_push_update -A "mwan3_policy_$1" \
+                                                 -m mark --mark 0x0/$MMX_MASK \
+                                                 -m comment --comment "blackhole" \
+                                                 -j MARK --set-xmark $MMX_BLACKHOLE/$MMX_MASK
+                               ;;
                        default)
-                               $IPT -A "mwan3_policy_$1" \
-                                       -m mark --mark 0x0/$MMX_MASK \
-                                       -m comment --comment "default" \
-                                       -j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
-                       ;;
+                               mwan3_push_update -A "mwan3_policy_$1" \
+                                                 -m mark --mark 0x0/$MMX_MASK \
+                                                 -m comment --comment "default" \
+                                                 -j MARK --set-xmark $MMX_DEFAULT/$MMX_MASK
+                               ;;
                        *)
-                               $IPT -A "mwan3_policy_$1" \
-                                       -m mark --mark 0x0/$MMX_MASK \
-                                       -m comment --comment "unreachable" \
-                                       -j MARK --set-xmark $MMX_UNREACHABLE/$MMX_MASK
-                       ;;
+                               mwan3_push_update -A "mwan3_policy_$1" \
+                                                 -m mark --mark 0x0/$MMX_MASK \
+                                                 -m comment --comment "unreachable" \
+                                                 -j MARK --set-xmark $MMX_UNREACHABLE/$MMX_MASK
+                               ;;
                esac
+               mwan3_push_update COMMIT
+               mwan3_push_update ""
+               if [ "$IPT" = "$IPT4" ]; then
+                       error=$(echo "$update" | $IPT4R 2>&1) || LOG error "create_policies_iptables ($1): $error"
+               else
+                       error=$(echo "$update" | $IPT6R 2>&1) || LOG error "create_policies_iptables ($1): $error"
+               fi
        done
 
        lowest_metric_v4=$DEFAULT_LOWEST_METRIC
@@ -884,27 +909,21 @@ mwan3_set_policies_iptables()
 mwan3_set_sticky_iptables()
 {
        local id iface
-
-       for iface in $($IPT4 -S "$policy" | cut -s -d'"' -f2 | awk '{print $1}'); do
-
+       for iface in $(echo "$current" | grep "^-A $policy" | cut -s -d'"' -f2 | awk '{print $1}'); do
                if [ "$iface" = "$1" ]; then
 
                        mwan3_get_iface_id id "$1"
 
                        [ -n "$id" ] || return 0
-
-                       for IPT in "$IPT4" "$IPT6"; do
-                               [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continuea
-                               if [ -n "$($IPT -S "mwan3_iface_in_$1" 2> /dev/null)" ]; then
-                                       $IPT -I "mwan3_rule_$rule" \
-                                               -m mark --mark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK \
-                                               -m set ! --match-set "mwan3_sticky_$rule" src,src \
-                                               -j MARK --set-xmark 0x0/$MMX_MASK
-                                       $IPT -I "mwan3_rule_$rule" \
-                                               -m mark --mark 0/$MMX_MASK \
-                                               -j MARK --set-xmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK
-                               fi
-                       done
+                       if [ -z "${current##*-N mwan3_iface_in_$1*}" ]; then
+                               mwan3_push_update -I "mwan3_rule_$rule" \
+                                                 -m mark --mark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK \
+                                                 -m set ! --match-set "mwan3_sticky_$rule" src,src \
+                                                 -j MARK --set-xmark 0x0/$MMX_MASK
+                               mwan3_push_update -I "mwan3_rule_$rule" \
+                                                 -m mark --mark 0/$MMX_MASK \
+                                                 -j MARK --set-xmark $(mwan3_id2mask id MMX_MASK)/$MMX_MASK
+                       fi
                fi
        done
 }
@@ -912,161 +931,209 @@ mwan3_set_sticky_iptables()
 mwan3_set_user_iptables_rule()
 {
        local ipset family proto policy src_ip src_port src_iface src_dev
-       local sticky dest_ip dest_port use_policy timeout rule policy IPT
-       local global_logging rule_logging loglevel
+       local sticky dest_ip dest_port use_policy timeout policy
+       local global_logging rule_logging loglevel rule_policy  rule ipv
 
        rule="$1"
-
+       ipv="$2"
+       rule_policy=0
        config_get sticky "$1" sticky 0
        config_get timeout "$1" timeout 600
        config_get ipset "$1" ipset
        config_get proto "$1" proto all
        config_get src_ip "$1" src_ip
        config_get src_iface "$1" src_iface
-       network_get_device src_dev "$src_iface"
        config_get src_port "$1" src_port
        config_get dest_ip "$1" dest_ip
        config_get dest_port "$1" dest_port
        config_get use_policy "$1" use_policy
        config_get family "$1" family any
+       config_get rule_logging "$1" logging 0
+       config_get global_logging globals logging 0
+       config_get loglevel globals loglevel notice
+
+       if [ -n "$src_iface" ]; then
+               network_get_device src_dev "$src_iface"
+               if [ -z "$src_dev" ]; then
+                       LOG notice "could not find device corresponding to src_iface $src_iface for rule $1"
+                       return
+               fi
+       fi
 
        [ -z "$dest_ip" ] && unset dest_ip
        [ -z "$src_ip" ] && unset src_ip
        [ -z "$ipset" ] && unset ipset
        [ -z "$src_port" ]  && unset src_port
        [ -z "$dest_port" ]  && unset dest_port
-       [ "$proto"  != 'tcp' ]  && [ "$proto" != 'udp' ] && {
+       if [ "$proto"  != 'tcp' ]  && [ "$proto" != 'udp' ]; then
                [ -n "$src_port" ] && {
-                       $LOG warn "src_port set to '$src_port' but proto set to '$proto' not tcp or udp. src_port will be ignored"
+                       LOG warn "src_port set to '$src_port' but proto set to '$proto' not tcp or udp. src_port will be ignored"
                }
+
                [ -n "$dest_port" ] && {
-                       $LOG warn "dest_port set to '$dest_port' but proto set to '$proto' not tcp or udp. dest_port will be ignored"
+                       LOG warn "dest_port set to '$dest_port' but proto set to '$proto' not tcp or udp. dest_port will be ignored"
                }
                unset src_port
                unset dest_port
-       }
-
-       config_get rule_logging "$1" logging 0
-       config_get global_logging globals logging 0
-       config_get loglevel globals loglevel notice
+       fi
 
        if [ "$1" != "$(echo "$1" | cut -c1-15)" ]; then
-               $LOG warn "Rule $1 exceeds max of 15 chars. Not setting rule" && return 0
+               LOG warn "Rule $1 exceeds max of 15 chars. Not setting rule" && return 0
        fi
 
        if [ -n "$ipset" ]; then
                ipset="-m set --match-set $ipset dst"
        fi
 
-       if [ -n "$use_policy" ]; then
-               if [ "$use_policy" = "default" ]; then
-                       policy="MARK --set-xmark $MMX_DEFAULT/$MMX_MASK"
-               elif [ "$use_policy" = "unreachable" ]; then
-                       policy="MARK --set-xmark $MMX_UNREACHABLE/$MMX_MASK"
-               elif [ "$use_policy" = "blackhole" ]; then
-                       policy="MARK --set-xmark $MMX_BLACKHOLE/$MMX_MASK"
-               else
-                       if [ "$sticky" -eq 1 ]; then
+       if [ -z "$use_policy" ]; then
+               return
+       fi
 
-                               policy="mwan3_policy_$use_policy"
+       if [ "$use_policy" = "default" ]; then
+               policy="MARK --set-xmark $MMX_DEFAULT/$MMX_MASK"
+       elif [ "$use_policy" = "unreachable" ]; then
+               policy="MARK --set-xmark $MMX_UNREACHABLE/$MMX_MASK"
+       elif [ "$use_policy" = "blackhole" ]; then
+               policy="MARK --set-xmark $MMX_BLACKHOLE/$MMX_MASK"
+       else
+               rule_policy=1
+               policy="mwan3_policy_$use_policy"
+               if [ "$sticky" -eq 1 ]; then
+                       $IPS -! create "mwan3_sticky_v4_$rule" \
+                            hash:ip,mark markmask "$MMX_MASK" \
+                            timeout "$timeout"
+                       [ $NO_IPV6 -eq 0 ] &&
+                               $IPS -! create "mwan3_sticky_v6_$rule" \
+                                    hash:ip,mark markmask "$MMX_MASK" \
+                                    timeout "$timeout" family inet6
+                       $IPS -! create "mwan3_sticky_$rule" list:set
+                       $IPS -! add "mwan3_sticky_$rule" "mwan3_sticky_v4_$rule"
+                       [ $NO_IPV6 -eq 0 ] &&
+                               $IPS -! add "mwan3_sticky_$rule" "mwan3_sticky_v6_$rule"
+               fi
+       fi
 
-                               for IPT in "$IPT4" "$IPT6"; do
-                                       [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
-                                       if ! $IPT -S "$policy" &> /dev/null; then
-                                               $IPT -N "$policy"
-                                       fi
+       [ "$ipv" = "ipv6" ] && [ $NO_IPV6 -ne 0 ] && return
+       [ "$family" = "ipv4" ] && [ "$ipv" = "ipv6" ] && return
+       [ "$family" = "ipv6" ] && [ "$ipv" = "ipv4" ] && return
 
-                                       if ! $IPT -S "mwan3_rule_$1" &> /dev/null; then
-                                               $IPT -N "mwan3_rule_$1"
-                                       fi
+       if [ $rule_policy -eq 1 ] && [ -n "${current##*-N $policy*}" ]; then
+               mwan3_push_update -N "$policy"
+       fi
 
-                                       $IPT -F "mwan3_rule_$1"
-                               done
+       if [ $rule_policy -eq 1 ] && [ "$sticky" -eq 1 ]; then
+               if [ -n "${current##*-N mwan3_rule_$1*}" ]; then
+                       mwan3_push_update -N "mwan3_rule_$1"
+               fi
 
-                               $IPS -! create "mwan3_sticky_v4_$rule" \
-                                       hash:ip,mark markmask "$MMX_MASK" \
-                                       timeout "$timeout"
-                               $IPS -! create "mwan3_sticky_v6_$rule" \
-                                       hash:ip,mark markmask "$MMX_MASK" \
-                                       timeout "$timeout" family inet6
-                               $IPS -! create "mwan3_sticky_$rule" list:set
-                               $IPS -! add "mwan3_sticky_$rule" "mwan3_sticky_v4_$rule"
-                               $IPS -! add "mwan3_sticky_$rule" "mwan3_sticky_v6_$rule"
+               mwan3_push_update -F "mwan3_rule_$1"
+               config_foreach mwan3_set_sticky_iptables interface $ipv
 
-                               config_foreach mwan3_set_sticky_iptables interface
-
-                               for IPT in "$IPT4" "$IPT6"; do
-                                       [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
-                                       $IPT -A "mwan3_rule_$1" \
-                                               -m mark --mark 0/$MMX_MASK \
-                                               -j "$policy"
-                                       $IPT -A "mwan3_rule_$1" \
-                                               -m mark ! --mark 0xfc00/0xfc00 \
-                                               -j SET --del-set "mwan3_sticky_$rule" src,src
-                                       $IPT -A "mwan3_rule_$1" \
-                                               -m mark ! --mark 0xfc00/0xfc00 \
-                                               -j SET --add-set "mwan3_sticky_$rule" src,src
-                               done
-
-                               policy="mwan3_rule_$1"
-                       else
-                               policy="mwan3_policy_$use_policy"
 
-                               for IPT in "$IPT4" "$IPT6"; do
-                                       [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
-                                       if ! $IPT -S "$policy" &> /dev/null; then
-                                               $IPT -N "$policy"
-                                       fi
-                               done
+               mwan3_push_update -A "mwan3_rule_$1" \
+                                 -m mark --mark 0/$MMX_MASK \
+                                 -j "$policy"
+               mwan3_push_update -A "mwan3_rule_$1" \
+                                 -m mark ! --mark 0xfc00/0xfc00 \
+                                 -j SET --del-set "mwan3_sticky_$rule" src,src
+               mwan3_push_update -A "mwan3_rule_$1" \
+                                 -m mark ! --mark 0xfc00/0xfc00 \
+                                 -j SET --add-set "mwan3_sticky_$rule" src,src
+               policy="mwan3_rule_$1"
+       fi
+       if [ "$global_logging" = "1" ] && [ "$rule_logging" = "1" ]; then
+               mwan3_push_update -A mwan3_rules \
+                                 -p "$proto" \
+                                 ${src_ip:+-s} $src_ip \
+                                 ${src_dev:+-i} $src_dev \
+                                 ${dest_ip:+-d} $dest_ip \
+                                 $ipset \
+                                 ${src_port:+-m} ${src_port:+multiport} ${src_port:+--sports} $src_port \
+                                 ${dest_port:+-m} ${dest_port:+multiport} ${dest_port:+--dports} $dest_port \
+                                 -m mark --mark 0/$MMX_MASK \
+                                 -m comment --comment "$1" \
+                                 -j LOG --log-level "$loglevel" --log-prefix "MWAN3($1)"
+       fi
 
-                       fi
-               fi
-               for IPT in "$IPT4" "$IPT6"; do
-                       [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
-                       [ "$family" = "ipv4" ] && [ "$IPT" = "$IPT6" ] && continue
-                       [ "$family" = "ipv6" ] && [ "$IPT" = "$IPT4" ] && continue
-                       [ "$global_logging" = "1" ] && [ "$rule_logging" = "1" ] && {
-                               $IPT -A mwan3_rules \
-                                    -p "$proto" \
-                                    ${src_ip:+-s} $src_ip \
-                                    ${src_dev:+-i} $src_dev \
-                                    ${dest_ip:+-d} $dest_ip \
-                                    $ipset \
-                                    ${src_port:+-m} ${src_port:+multiport} ${src_port:+--sports} $src_port \
-                                    ${dest_port:+-m} ${dest_port:+multiport} ${dest_port:+--dports} $dest_port \
-                                    -m mark --mark 0/$MMX_MASK \
-                                    -m comment --comment "$1" \
-                                    -j LOG --log-level "$loglevel" --log-prefix "MWAN3($1)"
-                       }
-
-                       $IPT -A mwan3_rules \
-                            -p "$proto" \
-                            ${src_ip:+-s} $src_ip \
-                            ${src_dev:+-i} $src_dev \
-                            ${dest_ip:+-d} $dest_ip \
-                            $ipset \
-                            ${src_port:+-m} ${src_port:+multiport} ${src_port:+--sports} $src_port \
-                            ${dest_port:+-m} ${dest_port:+multiport} ${dest_port:+--dports} $dest_port \
-                            -m mark --mark 0/$MMX_MASK \
-                            -j $policy
-               done
+       mwan3_push_update -A mwan3_rules \
+                         -p "$proto" \
+                         ${src_ip:+-s} $src_ip \
+                         ${src_dev:+-i} $src_dev \
+                         ${dest_ip:+-d} $dest_ip \
+                         $ipset \
+                         ${src_port:+-m} ${src_port:+multiport} ${src_port:+--sports} $src_port \
+                         ${dest_port:+-m} ${dest_port:+multiport} ${dest_port:+--dports} $dest_port \
+                         -m mark --mark 0/$MMX_MASK \
+                         -j $policy
+
+}
+
+mwan3_set_user_iface_rules()
+{
+       local current iface update family error device is_src_iface
+       iface=$1
+       device=$2
+
+       if [ -z "$device" ]; then
+               LOG notice "set_user_iface_rules: could not find device corresponding to iface $iface"
+               return
+       fi
+
+       config_get family "$iface" family ipv4
+
+       if [ "$family" = "ipv4" ]; then
+               IPT="$IPT4"
+               IPTR="$IPT4R"
+       elif [ "$family" = "ipv6" ]; then
+               IPT="$IPT6"
+               IPTR="$IPT6R"
        fi
+       $IPT -S | grep -q "^-A mwan3_rules.*-i $device" && return
+
+       is_src_iface=0
+
+       iface_rule()
+       {
+               local src_iface
+               config_get src_iface "$1" src_iface
+               [ "$src_iface" = "$iface" ] && is_src_iface=1
+       }
+       config_foreach iface_rule rule
+       [ $is_src_iface -eq 1 ] && mwan3_set_user_rules
 }
 
 mwan3_set_user_rules()
 {
-       local IPT
+       local IPT IPTR ipv
+       local current update error
 
-       for IPT in "$IPT4" "$IPT6"; do
-               [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
-               if ! $IPT -S mwan3_rules &> /dev/null; then
-                       $IPT -N mwan3_rules
+       for ipv in ipv4 ipv6; do
+               if [ "$ipv" = "ipv4" ]; then
+                       IPT="$IPT4"
+                       IPTR="$IPT4R"
+               elif [ "$ipv" = "ipv6" ]; then
+                       IPT="$IPT6"
+                       IPTR="$IPT6R"
                fi
+               [ "$ipv" = "ipv6" ] && [ $NO_IPV6 -ne 0 ] && continue
+               update="*mangle"
+               current="$($IPT -S)"
+
+
+               if [ -n "${current##*-N mwan3_rules*}" ]; then
+                       mwan3_push_update -N "mwan3_rules"
+               fi
+
+               mwan3_push_update -F mwan3_rules
 
-               $IPT -F mwan3_rules
+               config_foreach mwan3_set_user_iptables_rule rule "$ipv"
+
+               mwan3_push_update COMMIT
+               mwan3_push_update ""
+               error=$(echo "$update" | $IPTR 2>&1) || LOG error "set_user_rules: $error"
        done
 
-       config_foreach mwan3_set_user_iptables_rule rule
+
 }
 
 mwan3_set_iface_hotplug_state() {
@@ -1104,9 +1171,9 @@ mwan3_report_iface_status()
        if [ -z "$id" ] || [ -z "$device" ]; then
                result="offline"
        elif [ -n "$($IP rule | awk '$1 == "'$(($id+1000)):'"')" ] && \
-               [ -n "$($IP rule | awk '$1 == "'$(($id+2000)):'"')" ] && \
-               [ -n "$($IPT -S mwan3_iface_in_$1 2> /dev/null)" ] && \
-               [ -n "$($IP route list table $id default dev $device 2> /dev/null)" ]; then
+                    [ -n "$($IP rule | awk '$1 == "'$(($id+2000)):'"')" ] && \
+                    [ -n "$($IPT -S mwan3_iface_in_$1 2> /dev/null)" ] && \
+                    [ -n "$($IP route list table $id default dev $device 2> /dev/null)" ]; then
                json_init
                json_add_string section interfaces
                json_add_string interface "$1"
@@ -1120,9 +1187,9 @@ mwan3_report_iface_status()
                uptime="$(printf '%02dh:%02dm:%02ds\n' $(($uptime/3600)) $(($uptime%3600/60)) $(($uptime%60)))"
                result="$(mwan3_get_iface_hotplug_state $1) $online, uptime $uptime"
        elif [ -n "$($IP rule | awk '$1 == "'$(($id+1000)):'"')" ] || \
-               [ -n "$($IP rule | awk '$1 == "'$(($id+2000)):'"')" ] || \
-               [ -n "$($IPT -S mwan3_iface_in_$1 2> /dev/null)" ] || \
-               [ -n "$($IP route list table $id default dev $device 2> /dev/null)" ]; then
+                    [ -n "$($IP rule | awk '$1 == "'$(($id+2000)):'"')" ] || \
+                    [ -n "$($IPT -S mwan3_iface_in_$1 2> /dev/null)" ] || \
+                    [ -n "$($IP route list table $id default dev $device 2> /dev/null)" ]; then
                result="error"
        elif [ "$enabled" = "1" ]; then
                result="offline"
@@ -1228,7 +1295,7 @@ mwan3_flush_conntrack()
 
                if [ "$action" = "$flush_conntrack" ]; then
                        echo f > ${CONNTRACK_FILE}
-                       $LOG info "Connection tracking flushed for interface '$interface' on action '$action'"
+                       LOG info "Connection tracking flushed for interface '$interface' on action '$action'"
                fi
        }
 
@@ -1240,9 +1307,5 @@ mwan3_flush_conntrack()
 mwan3_track_clean()
 {
        rm -rf "$MWAN3_STATUS_DIR/${1}" &> /dev/null
-       [ -d "$MWAN3_STATUS_DIR" ] && {
-               if [ -z "$(ls -A "$MWAN3_STATUS_DIR")" ]; then
-                       rm -rf "$MWAN3_STATUS_DIR"
-               fi
-       }
+       rmdir --ignore-fail-on-non-empty "$MWAN3_STATUS_DIR"
 }
index 392d7f5003b675d5b509bddfbb084cdac459db70..fd6b5204f3db53089d7ef7255c9b23b57158d262 100755 (executable)
@@ -4,6 +4,7 @@
 . /usr/share/libubox/jshn.sh
 . /lib/functions/network.sh
 . /lib/mwan3/mwan3.sh
+. /lib/mwan3/common.sh
 
 help()
 {
@@ -37,52 +38,64 @@ ifdown()
 
        ACTION=ifdown INTERFACE=$1 /sbin/hotplug-call iface
 
-       kill $(pgrep -f "mwan3track $1 $2") &> /dev/null
+       kill $(pgrep -f "mwan3track $1 ") &> /dev/null
        mwan3_track_clean $1
 }
 
 ifup()
 {
-       local device enabled up l3_device status
-
-       mwan3_lock "command" "mwan3"
-
-       config_load mwan3
-       config_get_bool enabled globals 'enabled' 0
-
-       [ ${enabled} -gt 0 ] || {
-               echo "The service mwan3 is global disabled."
-               echo "Please execute \"/etc/init.d/mwan3 start\" first."
-               mwan3_unlock "command" "mwan3"
-               exit 1
-       }
+       local device enabled up l3_device status interface true_iface
 
        if [ -z "$1" ]; then
                echo "Expecting interface. Usage: mwan3 ifup <interface>"
-               mwan3_unlock "command" "mwan3"
                exit 0
        fi
 
        if [ -n "$2" ]; then
                echo "Too many arguments. Usage: mwan3 ifup <interface>"
-               mwan3_unlock "command" "mwan3"
                exit 0
        fi
 
-       config_get enabled "$1" enabled 0
-       mwan3_unlock "command" "mwan3"
+       interface=$1
+
+       if [ "${MWAN3_STARTUP}" != 1 ]; then
+               # It is not necessary to obtain a lock here, because it is obtained in the hotplug
+               # script, but we still want to do the check to print a useful error message
+               config_load mwan3
+               config_get_bool enabled globals 'enabled' 0
+
+               [ ${enabled} -gt 0 ] || {
+                       echo "The service mwan3 is global disabled."
+                       echo "Please execute \"/etc/init.d/mwan3 start\" first."
+                       exit 1
+               }
+       else
+               enabled=1
+       fi
+       mwan3_get_true_iface true_iface $interface
+       status=$(ubus -S call network.interface.$true_iface status)
 
-       status=$(ubus -S call network.interface.$1 status)
        [ -n "$status" ] && {
                json_load "$status"
                json_get_vars up l3_device
        }
+       hotplug_startup()
+       {
+               MWAN3_STARTUP=$MWAN3_STARTUP ACTION=ifup INTERFACE=$interface DEVICE=$l3_device TRUE_INTERFACE=$true_iface sh /etc/hotplug.d/iface/15-mwan3
+               MWAN3_STARTUP=$MWAN3_STARTUP ACTION=ifup INTERFACE=$interface DEVICE=$l3_device TRUE_INTERFACE=$true_iface sh /etc/hotplug.d/iface/16-mwan3-user
+       }
+
+       if [ "$up" != "1" ] || [ -z "$l3_device" ] || [ "$enabled" != "1" ]; then
+               return
+       fi
 
-       if [ "$up" = "1" ] \
-               && [ -n "$l3_device" ] \
-               && [ "$enabled" = "1" ]; then
-               ACTION=ifup INTERFACE=$1 DEVICE=$l3_device /sbin/hotplug-call iface
+       if [ "${MWAN3_STARTUP}" = 1 ]; then
+               hotplug_startup &
+               hotplug_pids="$hotplug_pids $!"
+       else
+               hotplug_startup
        fi
+
 }
 
 interfaces()
@@ -137,58 +150,78 @@ status()
 
 start()
 {
-       local enabled
-
+       local enabled hotplug_pids MWAN3_STARTUP
+       MWAN3_STARTUP=1
        mwan3_lock "command" "mwan3"
        uci_toggle_state mwan3 globals enabled "1"
-       mwan3_unlock "command" "mwan3"
-
        config_load mwan3
+
+       mwan3_update_iface_to_table
+       mwan3_set_connected_iptables
+       mwan3_set_custom_ipset
+       mwan3_set_general_rules
+       mwan3_set_general_iptables
        config_foreach ifup interface
+       wait $hotplug_pids
+       mwan3_add_all_nondefault_routes
+       mwan3_set_policies_iptables
+       mwan3_set_user_rules
+
+
+       mwan3_unlock "command" "mwan3"
        mwan3_rtmon
+       unset MWAN3_STARTUP
 }
 
 stop()
 {
-       local ipset route rule table IP IPT pid
+       local ipset rule IP IPTR IPT kill_pid family table tid
 
        mwan3_lock "command" "mwan3"
        uci_toggle_state mwan3 globals enabled "0"
 
-       kill -TERM $(pgrep -f "mwan3rtmon") > /dev/null 2>&1
-       kill -TERM $(pgrep -f "mwan3track") > /dev/null 2>&1
+       {
+               kill -TERM $(pgrep -f "mwan3rtmon") > /dev/null 2>&1
+               kill -TERM $(pgrep -f "mwan3track") > /dev/null 2>&1
 
-       sleep 1
-
-       kill -KILL $(pgrep -f "mwan3rtmon") > /dev/null 2>&1
-       kill -KILL $(pgrep -f "mwan3track") > /dev/null 2>&1
+               sleep 1
 
+               kill -KILL $(pgrep -f "mwan3rtmon") > /dev/null 2>&1
+               kill -KILL $(pgrep -f "mwan3track") > /dev/null 2>&1
+       } &
+       kill_pid=$!
        config_load mwan3
        config_foreach mwan3_track_clean interface
 
-       for IP in "$IP4" "$IP6"; do
-               [ "$IP" = "$IP6" ] && [ $NO_IPV6 -ne 0 ] && continue
-               for route in $(seq 1 $MWAN3_INTERFACE_MAX); do
-                       $IP route flush table $route &> /dev/null
+       for family in ipv4 ipv6; do
+               if [ "$family" = "ipv4" ]; then
+                       IPT="$IPT4"
+                       IPTR="$IPT4R"
+                       IP="$IP4"
+               elif [ "$family" = "ipv6" ]; then
+                       [ $NO_IPV6 -ne 0 ] && continue
+                       IPT="$IPT6"
+                       IPTR="$IPT6R"
+                       IP="$IP6"
+               fi
+
+               for tid in $(ip route list table all | sed -ne  's/.*table \([0-9]\+\).*/\1/p'|sort -u); do
+                       [ $tid -gt $MWAN3_INTERFACE_MAX ] && continue
+                       $IP route flush table $tid &> /dev/null
                done
 
                for rule in $($IP rule list | egrep '^[1-2][0-9]{3}\:' | cut -d ':' -f 1); do
                        $IP rule del pref $rule &> /dev/null
                done
-       done
-
-       for IPT in "$IPT4" "$IPT6"; do
-               [ "$IPT" = "$IPT6" ] && [ $NO_IPV6 -ne 0 ] && continue
-               $IPT -D PREROUTING -j mwan3_hook &> /dev/null
-               $IPT -D OUTPUT -j mwan3_hook &> /dev/null
-
-               for table in $($IPT -S | awk '{print $2}' | grep mwan3 | sort -u); do
-                       $IPT -F $table &> /dev/null
-               done
-
-               for table in $($IPT -S | awk '{print $2}' | grep mwan3 | sort -u); do
-                       $IPT -X $table &> /dev/null
-               done
+               table="$($IPT -S)"
+               {
+                       echo "*mangle";
+                       [ -z "${table##*PREROUTING -j mwan3_hook*}" ] && echo "-D PREROUTING -j mwan3_hook"
+                       [ -z "${table##*OUTPUT -j mwan3_hook*}" ] && echo "-D OUTPUT -j mwan3_hook"
+                       echo "$table" | awk '{print "-F "$2}' | grep mwan3 | sort -u
+                       echo "$table" | awk '{print "-X "$2}' | grep mwan3 | sort -u
+                       echo "COMMIT"
+               } | $IPTR
        done
 
        for ipset in $($IPS -n list | grep mwan3_); do
@@ -199,9 +232,19 @@ stop()
                $IPS -q destroy $ipset
        done
 
+       if ! pgrep -f "mwan3track" >/dev/null && ! pgrep -f "mwan3rtmon" >/dev/null; then
+               # mwan3track has already exited, no need to send
+               # TERM signal
+               kill $kill_pid 2>/dev/null
+       else
+               # mwan3track has not exited, wait for the killer
+               # to do its work
+               wait $kill_pid
+       fi
+       rm -rf $MWAN3_STATUS_DIR $MWAN3TRACK_STATUS_DIR
+
        mwan3_unlock "command" "mwan3"
 
-       rm -rf $MWAN3_STATUS_DIR $MWAN3TRACK_STATUS_DIR
 }
 
 restart() {
index 1075041bf6648eb6f841894a3466cab7682fd908..98a5c4b897a18d3b93fee5cf7ba16fabcdee8d7e 100755 (executable)
@@ -1,8 +1,9 @@
 #!/bin/sh
+
 . /lib/functions.sh
 . /lib/functions/network.sh
 . /lib/mwan3/mwan3.sh
-
+. /lib/mwan3/common.sh
 
 mwan3_rtmon_route_handle()
 {
index a4ced8255d6fe3000c993753f44f7ab3f30cea71..5cbf5f75d2ee8a17945a5d3b01e7fcb6032e39bd 100755 (executable)
 . /lib/functions.sh
 . /lib/mwan3/common.sh
 
-LOG="logger -t $(basename "$0")[$$] -p"
 INTERFACE=""
 DEVICE=""
 PING="/bin/ping"
 
 IFDOWN_EVENT=0
+IFUP_EVENT=0
 
 clean_up() {
-       $LOG notice "Stopping mwan3track for interface \"${INTERFACE}\""
+       LOG notice "Stopping mwan3track for interface \"${INTERFACE}\""
        exit 0
 }
 
 if_down() {
-       $LOG info "Detect ifdown event on interface ${INTERFACE} (${DEVICE})"
+       LOG info "Detect ifdown event on interface ${INTERFACE} (${DEVICE})"
        IFDOWN_EVENT=1
 }
 
+if_up() {
+       LOG info "Detect ifup event on interface ${INTERFACE} (${DEVICE})"
+       IFUP_EVENT=1
+}
+
 validate_track_method() {
        case "$1" in
                ping)
-                       command -v ping 1>/dev/null 2>&1 || {
-                               $LOG warn "Missing ping. Please install iputils-ping package or enable ping util and recompile busybox."
+                       [ -x "$PING" ] || {
+                               LOG warn "Missing ping. Please enable ping util and recompile busybox."
                                return 1
                        }
                        ;;
                arping)
                        command -v arping 1>/dev/null 2>&1 || {
-                               $LOG warn "Missing arping. Please install iputils-arping package."
+                               LOG warn "Missing arping. Please install iputils-arping package."
                                return 1
                        }
                        ;;
                httping)
                        command -v httping 1>/dev/null 2>&1 || {
-                               $LOG warn "Missing httping. Please install httping package."
+                               LOG warn "Missing httping. Please install httping package."
                                return 1
                        }
                        [ -n "$2" -a "$2" != "0.0.0.0" -a "$2" != "::" ] || {
-                               $LOG warn "Cannot determine source IP for the interface which is required by httping."
+                               LOG warn "Cannot determine source IP for the interface which is required by httping."
                                return 1
                        }
                        ;;
                nping-*)
                        command -v nping 1>/dev/null 2>&1 || {
-                               $LOG warn "Missing nping. Please install nping package."
+                               LOG warn "Missing nping. Please install nping package."
                                return 1
                        }
                        ;;
                *)
-                       $LOG warn "Unsupported tracking method: $track_method"
+                       LOG warn "Unsupported tracking method: $track_method"
                        return 2
                        ;;
        esac
 }
 
+disconnected() {
+       echo "offline" > /var/run/mwan3track/$INTERFACE/STATUS
+       echo "$(get_uptime)" > /var/run/mwan3track/$INTERFACE/OFFLINE
+       echo "0" > /var/run/mwan3track/$INTERFACE/ONLINE
+       score=0
+       [ "$1" == 1 ] && return
+       LOG notice "Interface $INTERFACE ($DEVICE) is offline"
+       env -i ACTION="disconnected" INTERFACE="$INTERFACE" DEVICE="$DEVICE" /sbin/hotplug-call iface
+}
+
+connected() {
+       echo "online" > /var/run/mwan3track/$INTERFACE/STATUS
+       echo "0" > /var/run/mwan3track/$INTERFACE/OFFLINE
+       echo "$(get_uptime)" > /var/run/mwan3track/$INTERFACE/ONLINE
+       host_up_count=0
+       lost=0
+       turn=0
+       loss=0
+       [ "$1" == 1 ] && return
+       LOG notice "Interface $INTERFACE ($DEVICE) is online"
+       env -i ACTION="connected" INTERFACE="$INTERFACE" DEVICE="$DEVICE" /sbin/hotplug-call iface
+}
+
+firstconnect() {
+       if [ "$STATUS" = "offline" ]; then
+               disconnected 1
+       else
+               connected 1
+       fi
+}
+
+update_status() {
+       local status track_ip
+       track_ip=$1
+       status=$2
+
+       echo "$1" > /var/run/mwan3track/$INTERFACE/TRACK_${track_ip}
+       [ -z "$3" ] && return
+       echo "$3" > /var/run/mwan3track/$INTERFACE/LATENCY_${track_ip}
+       echo "$4" > /var/run/mwan3track/$INTERFACE/LOSS_${track_ip}
+}
+
 main() {
        local reliability count timeout interval failure_interval
        local recovery_interval down up size
@@ -70,64 +117,49 @@ main() {
        DEVICE=$2
        STATUS=$3
        SRC_IP=$4
-       mkdir -p /var/run/mwan3track/$1
+       mkdir -p /var/run/mwan3track/$INTERFACE
        trap clean_up TERM
        trap if_down USR1
+       trap if_up USR2
 
        config_load mwan3
-       config_get track_method $1 track_method ping
-       config_get_bool httping_ssl $1 httping_ssl 0
+       config_get track_method $INTERFACE track_method ping
+       config_get_bool httping_ssl $INTERFACE httping_ssl 0
        validate_track_method $track_method $SRC_IP || {
                track_method=ping
                if validate_track_method $track_method; then
-                       $LOG warn "Using ping to track interface $INTERFACE avaliability"
+                       LOG warn "Using ping to track interface $INTERFACE avaliability"
                else
-                       $LOG err "No track method avaliable"
+                       LOG err "No track method avaliable"
                        exit 1
                fi
        }
-       config_get reliability $1 reliability 1
-       config_get count $1 count 1
-       config_get timeout $1 timeout 4
-       config_get interval $1 interval 10
-       config_get down $1 down 5
-       config_get up $1 up 5
-       config_get size $1 size 56
-       config_get max_ttl $1 max_ttl 60
-       config_get failure_interval $1 failure_interval $interval
-       config_get_bool keep_failure_interval $1 keep_failure_interval 0
-       config_get recovery_interval $1 recovery_interval $interval
-       config_get_bool check_quality $1 check_quality 0
-       config_get failure_latency $1 failure_latency 1000
-       config_get recovery_latency $1 recovery_latency 500
-       config_get failure_loss $1 failure_loss 40
-       config_get recovery_loss $1 recovery_loss 10
+       config_get reliability $INTERFACE reliability 1
+       config_get count $INTERFACE count 1
+       config_get timeout $INTERFACE timeout 4
+       config_get interval $INTERFACE interval 10
+       config_get down $INTERFACE down 5
+       config_get up $INTERFACE up 5
+       config_get size $INTERFACE size 56
+       config_get max_ttl $INTERFACE max_ttl 60
+       config_get failure_interval $INTERFACE failure_interval $interval
+       config_get_bool keep_failure_interval $INTERFACE keep_failure_interval 0
+       config_get recovery_interval $INTERFACE recovery_interval $interval
+       config_get_bool check_quality $INTERFACE check_quality 0
+       config_get failure_latency $INTERFACE failure_latency 1000
+       config_get recovery_latency $INTERFACE recovery_latency 500
+       config_get failure_loss $INTERFACE failure_loss 40
+       config_get recovery_loss $INTERFACE recovery_loss 10
 
        local score=$(($down+$up))
        local track_ips=$(echo $* | cut -d ' ' -f 5-99)
        local host_up_count=0
        local lost=0
-       local sleep_time=0
        local turn=0
-       local result
        local ping_protocol=4
-       local ping_result
-       local ping_result_raw
-       local ping_status
-       local loss=0
-       local latency=0
+       local sleep_time result ping_result ping_result_raw  ping_status loss latency
 
-       if [ "$STATUS" = "offline" ]; then
-               echo "offline" > /var/run/mwan3track/$1/STATUS
-               echo "0" > /var/run/mwan3track/$1/ONLINE
-               echo "$(get_uptime)" > /var/run/mwan3track/$1/OFFLINE
-               score=0
-       else
-               echo "online" > /var/run/mwan3track/$1/STATUS
-               echo "0" > /var/run/mwan3track/$1/OFFLINE
-               echo "$(get_uptime)" > /var/run/mwan3track/$1/ONLINE
-               env -i ACTION="connected" INTERFACE="$1" DEVICE="$2" /sbin/hotplug-call iface
-       fi
+       firstconnect
        while true; do
 
                sleep_time=$interval
@@ -139,16 +171,16 @@ main() {
                                                # pinging IPv6 hosts with an interface is troublesome
                                                # https://bugs.openwrt.org/index.php?do=details&task_id=2897
                                                # so get the IP address of the interface and use that instead
-                                               if echo $track_ip | grep -q ':'; then
-                                                       ADDR=$(ip -6 addr ls dev "$DEVICE" | sed -ne '/\/128/d' -e 's/ *inet6 \([^ \/]*\).* scope global.*/\1/p' | head -n1)
-                                                       [ -z "$ADDR" ] && ADDR=$(ip -6 addr ls dev "$DEVICE" | sed -ne 's/ *inet6 \([^ \/]*\).* scope global.*/\1/p')
+                                               if [ -z ${track_ip##*:*} ]; then
                                                        ping_protocol=6
+                                               else
+                                                       unset SRC_IP
                                                fi
                                                if [ $check_quality -eq 0 ]; then
-                                                       $PING -$ping_protocol -I ${ADDR:-$DEVICE} -c $count -W $timeout -s $size -t $max_ttl -q $track_ip &> /dev/null
+                                                       $PING -$ping_protocol -I ${SRC_IP:-$DEVICE} -c $count -W $timeout -s $size -t $max_ttl -q $track_ip &> /dev/null
                                                        result=$?
                                                else
-                                                       ping_result_raw="$($PING -$ping_protocol -I ${ADDR:-$DEVICE} -c $count -W $timeout -s $size -t $max_ttl -q $track_ip 2>/dev/null)"
+                                                       ping_result_raw="$($PING -$ping_protocol -I ${SRC_IP:-$DEVICE} -c $count -W $timeout -s $size -t $max_ttl -q $track_ip 2>/dev/null)"
                                                        ping_status=$?
                                                        ping_result=$(echo "$ping_result_raw" | tail -n2)
                                                        loss="$(echo "$ping_result" | grep "packet loss" |  cut -d "," -f3 | awk '{print $1}' | sed -e 's/%//')"
@@ -188,42 +220,40 @@ main() {
                                if [ $check_quality -eq 0 ]; then
                                        if [ $result -eq 0 ]; then
                                                let host_up_count++
-                                               echo "up" > /var/run/mwan3track/$1/TRACK_${track_ip}
+                                               update_status "$track_ip" "up"
+
                                                if [ $score -le $up ]; then
-                                                       $LOG info "Check ($track_method) success for target \"$track_ip\" on interface $1 ($2)"
+                                                       LOG info "Check ($track_method) success for target \"$track_ip\" on interface $INTERFACE ($DEVICE). Current score: $score"
                                                fi
                                        else
                                                let lost++
-                                               echo "down" > /var/run/mwan3track/$1/TRACK_${track_ip}
+                                               update_status "$track_ip" "down"
+
                                                if [ $score -gt $up ]; then
-                                                       $LOG info "Check ($track_method) failed for target \"$track_ip\" on interface $1 ($2)"
+                                                       LOG info "Check ($track_method) failed for target \"$track_ip\" on interface $INTERFACE ($DEVICE). Current score: $score"
                                                fi
                                        fi
                                else
                                        if [ "$loss" -ge "$failure_loss" -o "$latency" -ge "$failure_latency" ]; then
                                                let lost++
-                                               echo "down" > /var/run/mwan3track/$1/TRACK_${track_ip}
-                                               echo "$latency" > /var/run/mwan3track/$1/LATENCY_${track_ip}
-                                               echo "$loss" > /var/run/mwan3track/$1/LOSS_${track_ip}
+                                               update_status "$track_ip" "down" $latency $loss
 
                                                if [ $score -gt $up ]; then
-                                                       $LOG info "Check (${track_method}: latency=${latency}ms loss=${loss}%) failed for target \"$track_ip\" on interface $1 ($2)"
+                                                       LOG info "Check (${track_method}: latency=${latency}ms loss=${loss}%) failed for target \"$track_ip\" on interface $INTERFACE ($DEVICE). Current score: $score"
                                                fi
                                        elif [ "$loss" -le "$recovery_loss" -a "$latency" -le "$recovery_latency" ]; then
                                                let host_up_count++
-                                               echo "up" > /var/run/mwan3track/$1/TRACK_${track_ip}
-                                               echo "$latency" > /var/run/mwan3track/$1/LATENCY_${track_ip}
-                                               echo "$loss" > /var/run/mwan3track/$1/LOSS_${track_ip}
+                                               update_status "$track_ip" "up" $latency $loss
 
                                                if [ $score -le $up ]; then
-                                                       $LOG info "Check (${track_method}: latency=${latency}ms loss=${loss}%) success for target \"$track_ip\" on interface $1 ($2)"
+                                                       LOG info "Check (${track_method}: latency=${latency}ms loss=${loss}%) success for target \"$track_ip\" on interface $INTERFACE ($DEVICE). Current score: $score"
                                                fi
                                        else
-                                               echo "skipped" > /var/run/mwan3track/$1/TRACK_${track_ip}
+                                               echo "skipped" > /var/run/mwan3track/$INTERFACE/TRACK_${track_ip}
                                        fi
                                fi
                        else
-                               echo "skipped" > /var/run/mwan3track/$1/TRACK_${track_ip}
+                               echo "skipped" > /var/run/mwan3track/$INTERFACE/TRACK_${track_ip}
                        fi
                done
 
@@ -240,53 +270,50 @@ main() {
                        fi
 
                        if [ $score -eq $up ]; then
-                               echo "offline" > /var/run/mwan3track/$1/STATUS
-                               env -i ACTION=ifdown INTERFACE=$1 DEVICE=$2 /sbin/hotplug-call iface
+                               disconnected
                                score=0
                        fi
                else
                        if [ $score -lt $(($down+$up)) ] && [ $lost -gt 0 ]; then
-                               $LOG info "Lost $(($lost*$count)) ping(s) on interface $1 ($2)"
+                               LOG info "Lost $(($lost*$count)) ping(s) on interface $INTERFACE ($DEVICE). Current score: $score"
                        fi
 
                        let score++
                        lost=0
 
                        if [ $score -gt $up ]; then
-                               echo "online" > /var/run/mwan3track/$1/STATUS
+                               echo "online" > /var/run/mwan3track/$INTERFACE/STATUS
                                score=$(($down+$up))
                        elif [ $score -le $up ]; then
                                sleep_time=$recovery_interval
                        fi
 
                        if [ $score -eq $up ]; then
-                               $LOG notice "Interface $1 ($2) is online"
-                               echo "online" > /var/run/mwan3track/$1/STATUS
-                               env -i ACTION=ifup INTERFACE=$1 DEVICE=$2 /sbin/hotplug-call iface
-                               exit 0
+                               connected $INTERFACE $DEVICE
                        fi
                fi
 
                let turn++
                mkdir -p "/var/run/mwan3track/${1}"
-               echo "${lost}" > /var/run/mwan3track/$1/LOST
-               echo "${score}" > /var/run/mwan3track/$1/SCORE
-               echo "${turn}" > /var/run/mwan3track/$1/TURN
-               echo "$(get_uptime)" > /var/run/mwan3track/$1/TIME
+               echo "${lost}" > /var/run/mwan3track/$INTERFACE/LOST
+               echo "${score}" > /var/run/mwan3track/$INTERFACE/SCORE
+               echo "${turn}" > /var/run/mwan3track/$INTERFACE/TURN
+               echo "$(get_uptime)" > /var/run/mwan3track/$INTERFACE/TIME
 
                host_up_count=0
                sleep "${sleep_time}" &
                wait
 
                if [ "${IFDOWN_EVENT}" -eq 1 ]; then
-                       echo "offline" > /var/run/mwan3track/$1/STATUS
-                       echo "$(get_uptime)" > /var/run/mwan3track/$1/OFFLINE
-                       echo "0" > /var/run/mwan3track/$1/ONLINE
-                       $LOG notice "Interface $1 ($2) is offline"
-                       env -i ACTION="disconnected" INTERFACE="$1" DEVICE="$2" /sbin/hotplug-call iface
-                       score=0
+                       LOG debug "Register ifdown event on interface ${INTERFACE} (${DEVICE})"
+                       disconnected 1
                        IFDOWN_EVENT=0
                fi
+               if [ "${IFUP_EVENT}" -eq 1 ]; then
+                       LOG debug "Register ifup event on interface ${INTERFACE} (${DEVICE})"
+                       firstconnect
+                       IFUP_EVENT=0
+               fi
        done
 }