net/mlx4_en: Poll XDP TX completion queue in RX NAPI

Instead of having their own NAPIs, XDP TX completion queues get
polled within the corresponding RX NAPI.
This prevents any possible race on TX ring prod/cons indices,
between the context that issues the transmits (RX NAPI) and the
context that handles the completions (was previously done in
a separate NAPI).

This also improves performance, as it decreases the number
of NAPIs running on a CPU, saving the overhead of syncing
and switching between the contexts.

Performance tests:
Tested on ConnectX3Pro, Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz
Single queue no-RSS optimization ON.

XDP_TX packet rate:
-------------------------------------
     | Before    | After     | Gain |
IPv4 | 12.0 Mpps | 13.8 Mpps |  15% |
IPv6 | 12.0 Mpps | 13.8 Mpps |  15% |
-------------------------------------

Signed-off-by: Tariq Toukan <tariqt@mellanox.com>
Reviewed-by: Saeed Mahameed <saeedm@mellanox.com>
Cc: kernel-team@fb.com
Cc: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/mellanox/mlx4/en_cq.c b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
index 09dd3776db7632ae8818c9638507705176233ea4..85fe17e4dcfba44b8180488e2070793ab6ee00dd 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/en_cq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
@@ -146,16 +146,25 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
        if (err)
                goto free_eq;
 
-       cq->mcq.comp  = cq->type != RX ? mlx4_en_tx_irq : mlx4_en_rx_irq;
        cq->mcq.event = mlx4_en_cq_event;
 
-       if (cq->type != RX)
+       switch (cq->type) {
+       case TX:
+               cq->mcq.comp = mlx4_en_tx_irq;
                netif_tx_napi_add(cq->dev, &cq->napi, mlx4_en_poll_tx_cq,
                                  NAPI_POLL_WEIGHT);
-       else
+               napi_enable(&cq->napi);
+               break;
+       case RX:
+               cq->mcq.comp = mlx4_en_rx_irq;
                netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq, 64);
-
-       napi_enable(&cq->napi);
+               napi_enable(&cq->napi);
+               break;
+       case TX_XDP:
+               /* nothing regarding napi, it's shared with rx ring */
+               cq->xdp_busy = false;
+               break;
+       }
 
        return 0;
 
@@ -184,8 +193,10 @@ void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq)
 
 void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq)
 {
-       napi_disable(&cq->napi);
-       netif_napi_del(&cq->napi);
+       if (cq->type != TX_XDP) {
+               napi_disable(&cq->napi);
+               netif_napi_del(&cq->napi);
+       }
 
        mlx4_cq_free(priv->mdev->dev, &cq->mcq);
 }

diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index 51ce111b719e9b8fdaced81948c5fd62c8a93a13..99c02bb4f30202cd54462f2162fae6d140771f7b 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -1679,13 +1679,15 @@ int mlx4_en_start_port(struct net_device *dev)
                        if (t != TX_XDP) {
                                tx_ring->tx_queue = netdev_get_tx_queue(dev, i);
                                tx_ring->recycle_ring = NULL;
+
+                               /* Arm CQ for TX completions */
+                               mlx4_en_arm_cq(priv, cq);
+
                        } else {
                                mlx4_en_init_recycle_ring(priv, i);
+                               /* XDP TX CQ should never be armed */
                        }
 
-                       /* Arm CQ for TX completions */
-                       mlx4_en_arm_cq(priv, cq);
-
                        /* Set initial ownership of all Tx TXBBs to SW (1) */
                        for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE)
                                *((u32 *)(tx_ring->buf + j)) = 0xffffffff;

diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index 747e4d7d76932965c231fca431d00d6cd3871677..e5fb89505a134dbbfc54ee8af136432e9dfcf573 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -880,8 +880,10 @@ next:
        rcu_read_unlock();
 
        if (likely(polled)) {
-               if (doorbell_pending)
-                       mlx4_en_xmit_doorbell(priv->tx_ring[TX_XDP][cq->ring]);
+               if (doorbell_pending) {
+                       priv->tx_cq[TX_XDP][cq_ring]->xdp_busy = true;
+                       mlx4_en_xmit_doorbell(priv->tx_ring[TX_XDP][cq_ring]);
+               }
 
                mlx4_cq_set_ci(&cq->mcq);
                wmb(); /* ensure HW sees CQ consumer before we post new buffers */
@@ -912,16 +914,30 @@ int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
        struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
        struct net_device *dev = cq->dev;
        struct mlx4_en_priv *priv = netdev_priv(dev);
+       struct mlx4_en_cq *xdp_tx_cq = NULL;
+       bool clean_complete = true;
        int done;
 
+       if (priv->tx_ring_num[TX_XDP]) {
+               xdp_tx_cq = priv->tx_cq[TX_XDP][cq->ring];
+               if (xdp_tx_cq->xdp_busy) {
+                       clean_complete = mlx4_en_process_tx_cq(dev, xdp_tx_cq,
+                                                              budget);
+                       xdp_tx_cq->xdp_busy = !clean_complete;
+               }
+       }
+
        done = mlx4_en_process_rx_cq(dev, cq, budget);
 
        /* If we used up all the quota - we're probably not done yet... */
-       if (done == budget) {
+       if (done == budget || !clean_complete) {
                const struct cpumask *aff;
                struct irq_data *idata;
                int cpu_curr;
 
+               /* in case we got here because of !clean_complete */
+               done = budget;
+
                INC_PERF_COUNTER(priv->pstats.napi_quota);
 
                cpu_curr = smp_processor_id();

diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
index 01bb438792217992ce260b1255bcccd9f21b574f..500442c603426f475f5e450182f9c1980010c0b0 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
@@ -395,8 +395,8 @@ int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring)
        return cnt;
 }
 
-static bool mlx4_en_process_tx_cq(struct net_device *dev,
-                                 struct mlx4_en_cq *cq, int napi_budget)
+bool mlx4_en_process_tx_cq(struct net_device *dev,
+                          struct mlx4_en_cq *cq, int napi_budget)
 {
        struct mlx4_en_priv *priv = netdev_priv(dev);
        struct mlx4_cq *mcq = &cq->mcq;
@@ -1176,6 +1176,7 @@ netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring,
 
 tx_drop_count:
        rx_ring->xdp_tx_full++;
+       *doorbell_pending = true;
 tx_drop:
        return NETDEV_TX_BUSY;
 }

diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index c52edb717add9c035b9159e0f8b92d146649ad74..bde3bc869b7037101e8991a7d67f204f90b685f4 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -358,7 +358,10 @@ struct mlx4_en_cq {
        struct mlx4_hwq_resources wqres;
        int                     ring;
        struct net_device      *dev;
-       struct napi_struct      napi;
+       union {
+               struct napi_struct napi;
+               bool               xdp_busy;
+       };
        int size;
        int buf_size;
        int vector;
@@ -720,6 +723,8 @@ int mlx4_en_process_rx_cq(struct net_device *dev,
                          int budget);
 int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget);
 int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget);
+bool mlx4_en_process_tx_cq(struct net_device *dev,
+                          struct mlx4_en_cq *cq, int napi_budget);
 u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv,
                         struct mlx4_en_tx_ring *ring,
                         int index, u64 timestamp,