net: fix XPS static_key accounting

Commit 04157469b7b8 ("net: Use static_key for XPS maps") introduced a
static key for XPS, but the increments/decrements don't match.

First, the static key's counter is incremented once for each queue, but
only decremented once for a whole batch of queues, leading to large
unbalances.

Second, the xps_rxqs_needed key is decremented whenever we reset a batch
of queues, whether they had any rxqs mapping or not, so that if we setup
cpu-XPS on em1 and RXQS-XPS on em2, resetting the queues on em1 would
decrement the xps_rxqs_needed key.

This reworks the accounting scheme so that the xps_needed key is
incremented only once for each type of XPS for all the queues on a
device, and the xps_rxqs_needed key is incremented only once for all
queues. This is sufficient to let us retrieve queues via
get_xps_queue().

This patch introduces a new reset_xps_maps(), which reinitializes and
frees the appropriate map (xps_rxqs_map or xps_cpus_map), and drops a
reference to the needed keys:
 - both xps_needed and xps_rxqs_needed, in case of rxqs maps,
 - only xps_needed, in case of CPU maps.

Now, we also need to call reset_xps_maps() at the end of
__netif_set_xps_queue() when there's no active map left, for example
when writing '00000000,00000000' to all queues' xps_rxqs setting.

Fixes: 04157469b7b8 ("net: Use static_key for XPS maps")
Signed-off-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/core/dev.c b/net/core/dev.c
index 32a63f4c3a92efc906b68f4221c089befc8f48a9..3470e7fff1f47bb5105449584cb6f276eec9a38f 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2175,6 +2175,20 @@ static bool remove_xps_queue_cpu(struct net_device *dev,
        return active;
 }
 
+static void reset_xps_maps(struct net_device *dev,
+                          struct xps_dev_maps *dev_maps,
+                          bool is_rxqs_map)
+{
+       if (is_rxqs_map) {
+               static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
+               RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
+       } else {
+               RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
+       }
+       static_key_slow_dec_cpuslocked(&xps_needed);
+       kfree_rcu(dev_maps, rcu);
+}
+
 static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
                           struct xps_dev_maps *dev_maps, unsigned int nr_ids,
                           u16 offset, u16 count, bool is_rxqs_map)
@@ -2186,13 +2200,8 @@ static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
             j < nr_ids;)
                active |= remove_xps_queue_cpu(dev, dev_maps, j, offset,
                                               count);
-       if (!active) {
-               if (is_rxqs_map)
-                       RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
-               else
-                       RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
-               kfree_rcu(dev_maps, rcu);
-       }
+       if (!active)
+               reset_xps_maps(dev, dev_maps, is_rxqs_map);
 
        if (!is_rxqs_map) {
                for (i = offset + (count - 1); count--; i--) {
@@ -2236,10 +2245,6 @@ static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
                       false);
 
 out_no_maps:
-       if (static_key_enabled(&xps_rxqs_needed))
-               static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
-
-       static_key_slow_dec_cpuslocked(&xps_needed);
        mutex_unlock(&xps_map_mutex);
        cpus_read_unlock();
 }
@@ -2357,9 +2362,12 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
        if (!new_dev_maps)
                goto out_no_new_maps;
 
-       static_key_slow_inc_cpuslocked(&xps_needed);
-       if (is_rxqs_map)
-               static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
+       if (!dev_maps) {
+               /* Increment static keys at most once per type */
+               static_key_slow_inc_cpuslocked(&xps_needed);
+               if (is_rxqs_map)
+                       static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
+       }
 
        for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
             j < nr_ids;) {
@@ -2457,13 +2465,8 @@ out_no_new_maps:
        }
 
        /* free map if not active */
-       if (!active) {
-               if (is_rxqs_map)
-                       RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
-               else
-                       RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
-               kfree_rcu(dev_maps, rcu);
-       }
+       if (!active)
+               reset_xps_maps(dev, dev_maps, is_rxqs_map);
 
 out_no_maps:
        mutex_unlock(&xps_map_mutex);