--- /dev/null
+From 9cbfea02c1dbee0afb9128f065e6e793672b9ff7 Mon Sep 17 00:00:00 2001
+From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Date: Wed, 6 Jan 2021 22:42:02 +0800
+Subject: [PATCH 1/7] bcm63xx_enet: batch process rx path
+
+Use netif_receive_skb_list to batch process rx skb.
+Tested on BCM6328 320 MHz using iperf3 -M 512, increasing performance
+by 12.5%.
+
+Before:
+[ ID] Interval Transfer Bandwidth Retr
+[ 4] 0.00-30.00 sec 120 MBytes 33.7 Mbits/sec 277 sender
+[ 4] 0.00-30.00 sec 120 MBytes 33.5 Mbits/sec receiver
+
+After:
+[ ID] Interval Transfer Bandwidth Retr
+[ 4] 0.00-30.00 sec 136 MBytes 37.9 Mbits/sec 203 sender
+[ 4] 0.00-30.00 sec 135 MBytes 37.7 Mbits/sec receiver
+
+Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Acked-by: Florian Fainelli <f.fainelli@gmail.com>
+Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+---
+ drivers/net/ethernet/broadcom/bcm63xx_enet.c | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
++++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+@@ -298,10 +298,12 @@ static void bcm_enet_refill_rx_timer(str
+ static int bcm_enet_receive_queue(struct net_device *dev, int budget)
+ {
+ struct bcm_enet_priv *priv;
++ struct list_head rx_list;
+ struct device *kdev;
+ int processed;
+
+ priv = netdev_priv(dev);
++ INIT_LIST_HEAD(&rx_list);
+ kdev = &priv->pdev->dev;
+ processed = 0;
+
+@@ -392,10 +394,12 @@ static int bcm_enet_receive_queue(struct
+ skb->protocol = eth_type_trans(skb, dev);
+ dev->stats.rx_packets++;
+ dev->stats.rx_bytes += len;
+- netif_receive_skb(skb);
++ list_add_tail(&skb->list, &rx_list);
+
+ } while (--budget > 0);
+
++ netif_receive_skb_list(&rx_list);
++
+ if (processed || !priv->rx_desc_count) {
+ bcm_enet_refill_rx(dev);
+
--- /dev/null
+From 4c59b0f5543db80abbbe9efdd9b25e7899501db5 Mon Sep 17 00:00:00 2001
+From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Date: Wed, 6 Jan 2021 22:42:03 +0800
+Subject: [PATCH 2/7] bcm63xx_enet: add BQL support
+
+Add Byte Queue Limits support to reduce/remove bufferbloat in
+bcm63xx_enet.
+
+Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Acked-by: Florian Fainelli <f.fainelli@gmail.com>
+Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+---
+ drivers/net/ethernet/broadcom/bcm63xx_enet.c | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
++++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+@@ -418,9 +418,11 @@ static int bcm_enet_receive_queue(struct
+ static int bcm_enet_tx_reclaim(struct net_device *dev, int force)
+ {
+ struct bcm_enet_priv *priv;
++ unsigned int bytes;
+ int released;
+
+ priv = netdev_priv(dev);
++ bytes = 0;
+ released = 0;
+
+ while (priv->tx_desc_count < priv->tx_ring_size) {
+@@ -457,10 +459,13 @@ static int bcm_enet_tx_reclaim(struct ne
+ if (desc->len_stat & DMADESC_UNDER_MASK)
+ dev->stats.tx_errors++;
+
++ bytes += skb->len;
+ dev_kfree_skb(skb);
+ released++;
+ }
+
++ netdev_completed_queue(dev, released, bytes);
++
+ if (netif_queue_stopped(dev) && released)
+ netif_wake_queue(dev);
+
+@@ -627,6 +632,8 @@ bcm_enet_start_xmit(struct sk_buff *skb,
+ desc->len_stat = len_stat;
+ wmb();
+
++ netdev_sent_queue(dev, skb->len);
++
+ /* kick tx dma */
+ enet_dmac_writel(priv, priv->dma_chan_en_mask,
+ ENETDMAC_CHANCFG, priv->tx_chan);
+@@ -1170,6 +1177,7 @@ static int bcm_enet_stop(struct net_devi
+ kdev = &priv->pdev->dev;
+
+ netif_stop_queue(dev);
++ netdev_reset_queue(dev);
+ napi_disable(&priv->napi);
+ if (priv->has_phy)
+ phy_stop(dev->phydev);
+@@ -2343,6 +2351,7 @@ static int bcm_enetsw_stop(struct net_de
+
+ del_timer_sync(&priv->swphy_poll);
+ netif_stop_queue(dev);
++ netdev_reset_queue(dev);
+ napi_disable(&priv->napi);
+ del_timer_sync(&priv->rx_timeout);
+
--- /dev/null
+From 375281d3a6dcabaa98f489ee412aedca6d99dffb Mon Sep 17 00:00:00 2001
+From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Date: Wed, 6 Jan 2021 22:42:04 +0800
+Subject: [PATCH 3/7] bcm63xx_enet: add xmit_more support
+
+Support bulking hardware TX queue by using netdev_xmit_more().
+
+Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Acked-by: Florian Fainelli <f.fainelli@gmail.com>
+Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+---
+ drivers/net/ethernet/broadcom/bcm63xx_enet.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
++++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+@@ -635,7 +635,8 @@ bcm_enet_start_xmit(struct sk_buff *skb,
+ netdev_sent_queue(dev, skb->len);
+
+ /* kick tx dma */
+- enet_dmac_writel(priv, priv->dma_chan_en_mask,
++ if (!netdev_xmit_more() || !priv->tx_desc_count)
++ enet_dmac_writel(priv, priv->dma_chan_en_mask,
+ ENETDMAC_CHANCFG, priv->tx_chan);
+
+ /* stop queue if no more desc available */
--- /dev/null
+From c4a207865e7ea310dc146ff4aa1b0aa0c78d3fe1 Mon Sep 17 00:00:00 2001
+From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Date: Wed, 6 Jan 2021 22:42:05 +0800
+Subject: [PATCH 4/7] bcm63xx_enet: alloc rx skb with NET_IP_ALIGN
+
+Use netdev_alloc_skb_ip_align on newer SoCs with integrated switch
+(enetsw) when refilling RX. Increases packet processing performance
+by 30% (with netif_receive_skb_list).
+
+Non-enetsw SoCs cannot function with the extra pad so continue to use
+the regular netdev_alloc_skb.
+
+Tested on BCM6328 320 MHz and iperf3 -M 512 to measure packet/sec
+performance.
+
+Before:
+[ ID] Interval Transfer Bandwidth Retr
+[ 4] 0.00-30.00 sec 120 MBytes 33.7 Mbits/sec 277 sender
+[ 4] 0.00-30.00 sec 120 MBytes 33.5 Mbits/sec receiver
+
+After (+netif_receive_skb_list):
+[ 4] 0.00-30.00 sec 155 MBytes 43.3 Mbits/sec 354 sender
+[ 4] 0.00-30.00 sec 154 MBytes 43.1 Mbits/sec receiver
+
+Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Acked-by: Florian Fainelli <f.fainelli@gmail.com>
+Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+---
+ drivers/net/ethernet/broadcom/bcm63xx_enet.c | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
++++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+@@ -238,7 +238,10 @@ static int bcm_enet_refill_rx(struct net
+ desc = &priv->rx_desc_cpu[desc_idx];
+
+ if (!priv->rx_skb[desc_idx]) {
+- skb = netdev_alloc_skb(dev, priv->rx_skb_size);
++ if (priv->enet_is_sw)
++ skb = netdev_alloc_skb_ip_align(dev, priv->rx_skb_size);
++ else
++ skb = netdev_alloc_skb(dev, priv->rx_skb_size);
+ if (!skb)
+ break;
+ priv->rx_skb[desc_idx] = skb;
--- /dev/null
+From 3d0b72654b0c8304424503e7560ee8635dd56340 Mon Sep 17 00:00:00 2001
+From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Date: Wed, 6 Jan 2021 22:42:06 +0800
+Subject: [PATCH 5/7] bcm63xx_enet: consolidate rx SKB ring cleanup code
+
+The rx SKB ring use the same code for cleanup at various points.
+Combine them into a function to reduce lines of code.
+
+Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Acked-by: Florian Fainelli <f.fainelli@gmail.com>
+Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+---
+ drivers/net/ethernet/broadcom/bcm63xx_enet.c | 72 ++++++--------------
+ 1 file changed, 22 insertions(+), 50 deletions(-)
+
+--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
++++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+@@ -861,6 +861,24 @@ static void bcm_enet_adjust_link(struct
+ priv->pause_tx ? "tx" : "off");
+ }
+
++static void bcm_enet_free_rx_skb_ring(struct device *kdev, struct bcm_enet_priv *priv)
++{
++ int i;
++
++ for (i = 0; i < priv->rx_ring_size; i++) {
++ struct bcm_enet_desc *desc;
++
++ if (!priv->rx_skb[i])
++ continue;
++
++ desc = &priv->rx_desc_cpu[i];
++ dma_unmap_single(kdev, desc->address, priv->rx_skb_size,
++ DMA_FROM_DEVICE);
++ kfree_skb(priv->rx_skb[i]);
++ }
++ kfree(priv->rx_skb);
++}
++
+ /*
+ * open callback, allocate dma rings & buffers and start rx operation
+ */
+@@ -1085,18 +1103,7 @@ static int bcm_enet_open(struct net_devi
+ return 0;
+
+ out:
+- for (i = 0; i < priv->rx_ring_size; i++) {
+- struct bcm_enet_desc *desc;
+-
+- if (!priv->rx_skb[i])
+- continue;
+-
+- desc = &priv->rx_desc_cpu[i];
+- dma_unmap_single(kdev, desc->address, priv->rx_skb_size,
+- DMA_FROM_DEVICE);
+- kfree_skb(priv->rx_skb[i]);
+- }
+- kfree(priv->rx_skb);
++ bcm_enet_free_rx_skb_ring(kdev, priv);
+
+ out_free_tx_skb:
+ kfree(priv->tx_skb);
+@@ -1175,7 +1182,6 @@ static int bcm_enet_stop(struct net_devi
+ {
+ struct bcm_enet_priv *priv;
+ struct device *kdev;
+- int i;
+
+ priv = netdev_priv(dev);
+ kdev = &priv->pdev->dev;
+@@ -1204,20 +1210,9 @@ static int bcm_enet_stop(struct net_devi
+ bcm_enet_tx_reclaim(dev, 1);
+
+ /* free the rx skb ring */
+- for (i = 0; i < priv->rx_ring_size; i++) {
+- struct bcm_enet_desc *desc;
+-
+- if (!priv->rx_skb[i])
+- continue;
+-
+- desc = &priv->rx_desc_cpu[i];
+- dma_unmap_single(kdev, desc->address, priv->rx_skb_size,
+- DMA_FROM_DEVICE);
+- kfree_skb(priv->rx_skb[i]);
+- }
++ bcm_enet_free_rx_skb_ring(kdev, priv);
+
+ /* free remaining allocated memory */
+- kfree(priv->rx_skb);
+ kfree(priv->tx_skb);
+ dma_free_coherent(kdev, priv->rx_desc_alloc_size,
+ priv->rx_desc_cpu, priv->rx_desc_dma);
+@@ -2308,18 +2303,7 @@ static int bcm_enetsw_open(struct net_de
+ return 0;
+
+ out:
+- for (i = 0; i < priv->rx_ring_size; i++) {
+- struct bcm_enet_desc *desc;
+-
+- if (!priv->rx_skb[i])
+- continue;
+-
+- desc = &priv->rx_desc_cpu[i];
+- dma_unmap_single(kdev, desc->address, priv->rx_skb_size,
+- DMA_FROM_DEVICE);
+- kfree_skb(priv->rx_skb[i]);
+- }
+- kfree(priv->rx_skb);
++ bcm_enet_free_rx_skb_ring(kdev, priv);
+
+ out_free_tx_skb:
+ kfree(priv->tx_skb);
+@@ -2348,7 +2332,6 @@ static int bcm_enetsw_stop(struct net_de
+ {
+ struct bcm_enet_priv *priv;
+ struct device *kdev;
+- int i;
+
+ priv = netdev_priv(dev);
+ kdev = &priv->pdev->dev;
+@@ -2371,20 +2354,9 @@ static int bcm_enetsw_stop(struct net_de
+ bcm_enet_tx_reclaim(dev, 1);
+
+ /* free the rx skb ring */
+- for (i = 0; i < priv->rx_ring_size; i++) {
+- struct bcm_enet_desc *desc;
+-
+- if (!priv->rx_skb[i])
+- continue;
+-
+- desc = &priv->rx_desc_cpu[i];
+- dma_unmap_single(kdev, desc->address, priv->rx_skb_size,
+- DMA_FROM_DEVICE);
+- kfree_skb(priv->rx_skb[i]);
+- }
++ bcm_enet_free_rx_skb_ring(kdev, priv);
+
+ /* free remaining allocated memory */
+- kfree(priv->rx_skb);
+ kfree(priv->tx_skb);
+ dma_free_coherent(kdev, priv->rx_desc_alloc_size,
+ priv->rx_desc_cpu, priv->rx_desc_dma);
--- /dev/null
+From d27de0ef5ef995df2cc5f5c006c0efcf0a62b6af Mon Sep 17 00:00:00 2001
+From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Date: Wed, 6 Jan 2021 22:42:07 +0800
+Subject: [PATCH 6/7] bcm63xx_enet: convert to build_skb
+
+We can increase the efficiency of rx path by using buffers to receive
+packets then build SKBs around them just before passing into the network
+stack. In contrast, preallocating SKBs too early reduces CPU cache
+efficiency.
+
+Check if we're in NAPI context when refilling RX. Normally we're almost
+always running in NAPI context. Dispatch to napi_alloc_frag directly
+instead of relying on netdev_alloc_frag which does the same but
+with the overhead of local_bh_disable/enable.
+
+Tested on BCM6328 320 MHz and iperf3 -M 512 to measure packet/sec
+performance. Included netif_receive_skb_list and NET_IP_ALIGN
+optimizations.
+
+Before:
+[ ID] Interval Transfer Bandwidth Retr
+[ 4] 0.00-10.00 sec 49.9 MBytes 41.9 Mbits/sec 197 sender
+[ 4] 0.00-10.00 sec 49.3 MBytes 41.3 Mbits/sec receiver
+
+After:
+[ ID] Interval Transfer Bandwidth Retr
+[ 4] 0.00-30.00 sec 171 MBytes 47.8 Mbits/sec 272 sender
+[ 4] 0.00-30.00 sec 170 MBytes 47.6 Mbits/sec receiver
+
+Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Acked-by: Florian Fainelli <f.fainelli@gmail.com>
+Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+---
+ drivers/net/ethernet/broadcom/bcm63xx_enet.c | 111 ++++++++++---------
+ drivers/net/ethernet/broadcom/bcm63xx_enet.h | 14 ++-
+ 2 files changed, 71 insertions(+), 54 deletions(-)
+
+--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
++++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+@@ -221,7 +221,7 @@ static void bcm_enet_mdio_write_mii(stru
+ /*
+ * refill rx queue
+ */
+-static int bcm_enet_refill_rx(struct net_device *dev)
++static int bcm_enet_refill_rx(struct net_device *dev, bool napi_mode)
+ {
+ struct bcm_enet_priv *priv;
+
+@@ -229,29 +229,29 @@ static int bcm_enet_refill_rx(struct net
+
+ while (priv->rx_desc_count < priv->rx_ring_size) {
+ struct bcm_enet_desc *desc;
+- struct sk_buff *skb;
+- dma_addr_t p;
+ int desc_idx;
+ u32 len_stat;
+
+ desc_idx = priv->rx_dirty_desc;
+ desc = &priv->rx_desc_cpu[desc_idx];
+
+- if (!priv->rx_skb[desc_idx]) {
+- if (priv->enet_is_sw)
+- skb = netdev_alloc_skb_ip_align(dev, priv->rx_skb_size);
++ if (!priv->rx_buf[desc_idx]) {
++ void *buf;
++
++ if (likely(napi_mode))
++ buf = napi_alloc_frag(priv->rx_frag_size);
+ else
+- skb = netdev_alloc_skb(dev, priv->rx_skb_size);
+- if (!skb)
++ buf = netdev_alloc_frag(priv->rx_frag_size);
++ if (unlikely(!buf))
+ break;
+- priv->rx_skb[desc_idx] = skb;
+- p = dma_map_single(&priv->pdev->dev, skb->data,
+- priv->rx_skb_size,
+- DMA_FROM_DEVICE);
+- desc->address = p;
++ priv->rx_buf[desc_idx] = buf;
++ desc->address = dma_map_single(&priv->pdev->dev,
++ buf + priv->rx_buf_offset,
++ priv->rx_buf_size,
++ DMA_FROM_DEVICE);
+ }
+
+- len_stat = priv->rx_skb_size << DMADESC_LENGTH_SHIFT;
++ len_stat = priv->rx_buf_size << DMADESC_LENGTH_SHIFT;
+ len_stat |= DMADESC_OWNER_MASK;
+ if (priv->rx_dirty_desc == priv->rx_ring_size - 1) {
+ len_stat |= (DMADESC_WRAP_MASK >> priv->dma_desc_shift);
+@@ -291,7 +291,7 @@ static void bcm_enet_refill_rx_timer(str
+ struct net_device *dev = priv->net_dev;
+
+ spin_lock(&priv->rx_lock);
+- bcm_enet_refill_rx(dev);
++ bcm_enet_refill_rx(dev, false);
+ spin_unlock(&priv->rx_lock);
+ }
+
+@@ -321,6 +321,7 @@ static int bcm_enet_receive_queue(struct
+ int desc_idx;
+ u32 len_stat;
+ unsigned int len;
++ void *buf;
+
+ desc_idx = priv->rx_curr_desc;
+ desc = &priv->rx_desc_cpu[desc_idx];
+@@ -366,16 +367,14 @@ static int bcm_enet_receive_queue(struct
+ }
+
+ /* valid packet */
+- skb = priv->rx_skb[desc_idx];
++ buf = priv->rx_buf[desc_idx];
+ len = (len_stat & DMADESC_LENGTH_MASK) >> DMADESC_LENGTH_SHIFT;
+ /* don't include FCS */
+ len -= 4;
+
+ if (len < copybreak) {
+- struct sk_buff *nskb;
+-
+- nskb = napi_alloc_skb(&priv->napi, len);
+- if (!nskb) {
++ skb = napi_alloc_skb(&priv->napi, len);
++ if (unlikely(!skb)) {
+ /* forget packet, just rearm desc */
+ dev->stats.rx_dropped++;
+ continue;
+@@ -383,14 +382,21 @@ static int bcm_enet_receive_queue(struct
+
+ dma_sync_single_for_cpu(kdev, desc->address,
+ len, DMA_FROM_DEVICE);
+- memcpy(nskb->data, skb->data, len);
++ memcpy(skb->data, buf + priv->rx_buf_offset, len);
+ dma_sync_single_for_device(kdev, desc->address,
+ len, DMA_FROM_DEVICE);
+- skb = nskb;
+ } else {
+- dma_unmap_single(&priv->pdev->dev, desc->address,
+- priv->rx_skb_size, DMA_FROM_DEVICE);
+- priv->rx_skb[desc_idx] = NULL;
++ dma_unmap_single(kdev, desc->address,
++ priv->rx_buf_size, DMA_FROM_DEVICE);
++ priv->rx_buf[desc_idx] = NULL;
++
++ skb = build_skb(buf, priv->rx_frag_size);
++ if (unlikely(!skb)) {
++ skb_free_frag(buf);
++ dev->stats.rx_dropped++;
++ continue;
++ }
++ skb_reserve(skb, priv->rx_buf_offset);
+ }
+
+ skb_put(skb, len);
+@@ -404,7 +410,7 @@ static int bcm_enet_receive_queue(struct
+ netif_receive_skb_list(&rx_list);
+
+ if (processed || !priv->rx_desc_count) {
+- bcm_enet_refill_rx(dev);
++ bcm_enet_refill_rx(dev, true);
+
+ /* kick rx dma */
+ enet_dmac_writel(priv, priv->dma_chan_en_mask,
+@@ -861,22 +867,22 @@ static void bcm_enet_adjust_link(struct
+ priv->pause_tx ? "tx" : "off");
+ }
+
+-static void bcm_enet_free_rx_skb_ring(struct device *kdev, struct bcm_enet_priv *priv)
++static void bcm_enet_free_rx_buf_ring(struct device *kdev, struct bcm_enet_priv *priv)
+ {
+ int i;
+
+ for (i = 0; i < priv->rx_ring_size; i++) {
+ struct bcm_enet_desc *desc;
+
+- if (!priv->rx_skb[i])
++ if (!priv->rx_buf[i])
+ continue;
+
+ desc = &priv->rx_desc_cpu[i];
+- dma_unmap_single(kdev, desc->address, priv->rx_skb_size,
++ dma_unmap_single(kdev, desc->address, priv->rx_buf_size,
+ DMA_FROM_DEVICE);
+- kfree_skb(priv->rx_skb[i]);
++ skb_free_frag(priv->rx_buf[i]);
+ }
+- kfree(priv->rx_skb);
++ kfree(priv->rx_buf);
+ }
+
+ /*
+@@ -988,10 +994,10 @@ static int bcm_enet_open(struct net_devi
+ priv->tx_curr_desc = 0;
+ spin_lock_init(&priv->tx_lock);
+
+- /* init & fill rx ring with skbs */
+- priv->rx_skb = kcalloc(priv->rx_ring_size, sizeof(struct sk_buff *),
++ /* init & fill rx ring with buffers */
++ priv->rx_buf = kcalloc(priv->rx_ring_size, sizeof(void *),
+ GFP_KERNEL);
+- if (!priv->rx_skb) {
++ if (!priv->rx_buf) {
+ ret = -ENOMEM;
+ goto out_free_tx_skb;
+ }
+@@ -1008,8 +1014,8 @@ static int bcm_enet_open(struct net_devi
+ enet_dmac_writel(priv, ENETDMA_BUFALLOC_FORCE_MASK | 0,
+ ENETDMAC_BUFALLOC, priv->rx_chan);
+
+- if (bcm_enet_refill_rx(dev)) {
+- dev_err(kdev, "cannot allocate rx skb queue\n");
++ if (bcm_enet_refill_rx(dev, false)) {
++ dev_err(kdev, "cannot allocate rx buffer queue\n");
+ ret = -ENOMEM;
+ goto out;
+ }
+@@ -1103,7 +1109,7 @@ static int bcm_enet_open(struct net_devi
+ return 0;
+
+ out:
+- bcm_enet_free_rx_skb_ring(kdev, priv);
++ bcm_enet_free_rx_buf_ring(kdev, priv);
+
+ out_free_tx_skb:
+ kfree(priv->tx_skb);
+@@ -1209,8 +1215,8 @@ static int bcm_enet_stop(struct net_devi
+ /* force reclaim of all tx buffers */
+ bcm_enet_tx_reclaim(dev, 1);
+
+- /* free the rx skb ring */
+- bcm_enet_free_rx_skb_ring(kdev, priv);
++ /* free the rx buffer ring */
++ bcm_enet_free_rx_buf_ring(kdev, priv);
+
+ /* free remaining allocated memory */
+ kfree(priv->tx_skb);
+@@ -1637,9 +1643,12 @@ static int bcm_enet_change_mtu(struct ne
+ * align rx buffer size to dma burst len, account FCS since
+ * it's appended
+ */
+- priv->rx_skb_size = ALIGN(actual_mtu + ETH_FCS_LEN,
++ priv->rx_buf_size = ALIGN(actual_mtu + ETH_FCS_LEN,
+ priv->dma_maxburst * 4);
+
++ priv->rx_frag_size = SKB_DATA_ALIGN(priv->rx_buf_offset + priv->rx_buf_size) +
++ SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
++
+ dev->mtu = new_mtu;
+ return 0;
+ }
+@@ -1725,6 +1734,7 @@ static int bcm_enet_probe(struct platfor
+
+ priv->enet_is_sw = false;
+ priv->dma_maxburst = BCMENET_DMA_MAXBURST;
++ priv->rx_buf_offset = NET_SKB_PAD;
+
+ ret = bcm_enet_change_mtu(dev, dev->mtu);
+ if (ret)
+@@ -2142,7 +2152,7 @@ static int bcm_enetsw_open(struct net_de
+ priv->tx_skb = kcalloc(priv->tx_ring_size, sizeof(struct sk_buff *),
+ GFP_KERNEL);
+ if (!priv->tx_skb) {
+- dev_err(kdev, "cannot allocate rx skb queue\n");
++ dev_err(kdev, "cannot allocate tx skb queue\n");
+ ret = -ENOMEM;
+ goto out_free_tx_ring;
+ }
+@@ -2152,11 +2162,11 @@ static int bcm_enetsw_open(struct net_de
+ priv->tx_curr_desc = 0;
+ spin_lock_init(&priv->tx_lock);
+
+- /* init & fill rx ring with skbs */
+- priv->rx_skb = kcalloc(priv->rx_ring_size, sizeof(struct sk_buff *),
++ /* init & fill rx ring with buffers */
++ priv->rx_buf = kcalloc(priv->rx_ring_size, sizeof(void *),
+ GFP_KERNEL);
+- if (!priv->rx_skb) {
+- dev_err(kdev, "cannot allocate rx skb queue\n");
++ if (!priv->rx_buf) {
++ dev_err(kdev, "cannot allocate rx buffer queue\n");
+ ret = -ENOMEM;
+ goto out_free_tx_skb;
+ }
+@@ -2203,8 +2213,8 @@ static int bcm_enetsw_open(struct net_de
+ enet_dma_writel(priv, ENETDMA_BUFALLOC_FORCE_MASK | 0,
+ ENETDMA_BUFALLOC_REG(priv->rx_chan));
+
+- if (bcm_enet_refill_rx(dev)) {
+- dev_err(kdev, "cannot allocate rx skb queue\n");
++ if (bcm_enet_refill_rx(dev, false)) {
++ dev_err(kdev, "cannot allocate rx buffer queue\n");
+ ret = -ENOMEM;
+ goto out;
+ }
+@@ -2303,7 +2313,7 @@ static int bcm_enetsw_open(struct net_de
+ return 0;
+
+ out:
+- bcm_enet_free_rx_skb_ring(kdev, priv);
++ bcm_enet_free_rx_buf_ring(kdev, priv);
+
+ out_free_tx_skb:
+ kfree(priv->tx_skb);
+@@ -2353,8 +2363,8 @@ static int bcm_enetsw_stop(struct net_de
+ /* force reclaim of all tx buffers */
+ bcm_enet_tx_reclaim(dev, 1);
+
+- /* free the rx skb ring */
+- bcm_enet_free_rx_skb_ring(kdev, priv);
++ /* free the rx buffer ring */
++ bcm_enet_free_rx_buf_ring(kdev, priv);
+
+ /* free remaining allocated memory */
+ kfree(priv->tx_skb);
+@@ -2655,6 +2665,7 @@ static int bcm_enetsw_probe(struct platf
+ priv->rx_ring_size = BCMENET_DEF_RX_DESC;
+ priv->tx_ring_size = BCMENET_DEF_TX_DESC;
+ priv->dma_maxburst = BCMENETSW_DMA_MAXBURST;
++ priv->rx_buf_offset = NET_SKB_PAD + NET_IP_ALIGN;
+
+ pd = dev_get_platdata(&pdev->dev);
+ if (pd) {
+--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.h
++++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
+@@ -230,11 +230,17 @@ struct bcm_enet_priv {
+ /* next dirty rx descriptor to refill */
+ int rx_dirty_desc;
+
+- /* size of allocated rx skbs */
+- unsigned int rx_skb_size;
++ /* size of allocated rx buffers */
++ unsigned int rx_buf_size;
+
+- /* list of skb given to hw for rx */
+- struct sk_buff **rx_skb;
++ /* allocated rx buffer offset */
++ unsigned int rx_buf_offset;
++
++ /* size of allocated rx frag */
++ unsigned int rx_frag_size;
++
++ /* list of buffer given to hw for rx */
++ void **rx_buf;
+
+ /* used when rx skb allocation failed, so we defer rx queue
+ * refill */
--- /dev/null
+From ae2259eebeacb7753e3043278957b45840123972 Mon Sep 17 00:00:00 2001
+From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Date: Wed, 6 Jan 2021 22:42:08 +0800
+Subject: [PATCH 7/7] bcm63xx_enet: improve rx loop
+
+Use existing rx processed count to track against budget, thereby making
+budget decrement operation redundant.
+
+rx_desc_count can be calculated outside the rx loop, making the loop a
+bit smaller.
+
+Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
+Acked-by: Florian Fainelli <f.fainelli@gmail.com>
+Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+---
+ drivers/net/ethernet/broadcom/bcm63xx_enet.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
++++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+@@ -340,7 +340,6 @@ static int bcm_enet_receive_queue(struct
+ priv->rx_curr_desc++;
+ if (priv->rx_curr_desc == priv->rx_ring_size)
+ priv->rx_curr_desc = 0;
+- priv->rx_desc_count--;
+
+ /* if the packet does not have start of packet _and_
+ * end of packet flag set, then just recycle it */
+@@ -405,9 +404,10 @@ static int bcm_enet_receive_queue(struct
+ dev->stats.rx_bytes += len;
+ list_add_tail(&skb->list, &rx_list);
+
+- } while (--budget > 0);
++ } while (processed < budget);
+
+ netif_receive_skb_list(&rx_list);
++ priv->rx_desc_count -= processed;
+
+ if (processed || !priv->rx_desc_count) {
+ bcm_enet_refill_rx(dev, true);
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-@@ -1612,7 +1612,7 @@ static int bcm_enet_change_mtu(struct ne
+@@ -1629,7 +1629,7 @@ static int bcm_enet_change_mtu(struct ne
return -EBUSY;
/* add ethernet header + vlan tag size */
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-@@ -854,10 +854,8 @@ static int bcm_enet_open(struct net_devi
+@@ -893,10 +893,8 @@ static int bcm_enet_open(struct net_devi
struct bcm_enet_priv *priv;
struct sockaddr addr;
struct device *kdev;
void *p;
u32 val;
-@@ -865,31 +863,10 @@ static int bcm_enet_open(struct net_devi
+@@ -904,31 +902,10 @@ static int bcm_enet_open(struct net_devi
kdev = &priv->pdev->dev;
if (priv->has_phy) {
}
/* mask all interrupts and request them */
-@@ -899,7 +876,7 @@ static int bcm_enet_open(struct net_devi
+@@ -938,7 +915,7 @@ static int bcm_enet_open(struct net_devi
ret = request_irq(dev->irq, bcm_enet_isr_mac, 0, dev->name, dev);
if (ret)
ret = request_irq(priv->irq_rx, bcm_enet_isr_dma, 0,
dev->name, dev);
-@@ -1061,8 +1038,8 @@ static int bcm_enet_open(struct net_devi
+@@ -1100,8 +1077,8 @@ static int bcm_enet_open(struct net_devi
enet_dmac_writel(priv, priv->dma_chan_int_mask,
ENETDMAC_IRMASK, priv->tx_chan);
else
bcm_enet_adjust_link(dev);
-@@ -1103,10 +1080,6 @@ out_freeirq_rx:
+@@ -1131,10 +1108,6 @@ out_freeirq_rx:
out_freeirq:
free_irq(dev->irq, dev);
return ret;
}
-@@ -1211,10 +1184,6 @@ static int bcm_enet_stop(struct net_devi
+@@ -1228,10 +1201,6 @@ static int bcm_enet_stop(struct net_devi
free_irq(priv->irq_rx, dev);
free_irq(dev->irq, dev);
return 0;
}
-@@ -1779,14 +1748,47 @@ static int bcm_enet_probe(struct platfor
+@@ -1800,14 +1769,47 @@ static int bcm_enet_probe(struct platfor
/* do minimal hardware init to be able to probe mii bus */
bcm_enet_hw_preinit(priv);
}
bus = priv->mii_bus;
-@@ -1810,6 +1812,26 @@ static int bcm_enet_probe(struct platfor
+@@ -1831,6 +1833,26 @@ static int bcm_enet_probe(struct platfor
dev_err(&pdev->dev, "unable to register mdio bus\n");
goto out_free_mdio;
}
} else {
/* run platform code to initialize PHY device */
-@@ -1817,45 +1839,16 @@ static int bcm_enet_probe(struct platfor
+@@ -1838,45 +1860,16 @@ static int bcm_enet_probe(struct platfor
pd->mii_config(dev, 1, bcm_enet_mdio_read_mii,
bcm_enet_mdio_write_mii)) {
dev_err(&pdev->dev, "unable to configure mdio bus\n");
if (priv->mii_bus)
mdiobus_unregister(priv->mii_bus);
-@@ -1863,6 +1856,9 @@ out_free_mdio:
+@@ -1884,6 +1877,9 @@ out_free_mdio:
if (priv->mii_bus)
mdiobus_free(priv->mii_bus);
out_uninit_hw:
/* turn off mdc clock */
enet_writel(priv, 0, ENET_MIISC_REG);
-@@ -1893,6 +1889,7 @@ static int bcm_enet_remove(struct platfo
+@@ -1914,6 +1910,7 @@ static int bcm_enet_remove(struct platfo
enet_writel(priv, 0, ENET_MIISC_REG);
if (priv->has_phy) {
#define ENETSW_MDIOC_EXT_MASK (1 << 16)
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-@@ -2162,6 +2162,18 @@ static int bcm_enetsw_open(struct net_de
+@@ -2183,6 +2183,18 @@ static int bcm_enetsw_open(struct net_de
priv->sw_port_link[i] = 0;
}
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.h
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
-@@ -332,6 +332,9 @@ struct bcm_enet_priv {
+@@ -338,6 +338,9 @@ struct bcm_enet_priv {
struct bcm63xx_enetsw_port used_ports[ENETSW_MAX_PORT];
int sw_port_link[ENETSW_MAX_PORT];
#include <bcm63xx_dev_enet.h>
#include "bcm63xx_enet.h"
-@@ -1909,7 +1910,8 @@ static int bcm_enet_remove(struct platfo
+@@ -1930,7 +1931,8 @@ static int bcm_enet_remove(struct platfo
return 0;
}
.probe = bcm_enet_probe,
.remove = bcm_enet_remove,
.driver = {
-@@ -1918,6 +1920,42 @@ struct platform_driver bcm63xx_enet_driv
+@@ -1939,6 +1941,42 @@ struct platform_driver bcm63xx_enet_driv
},
};
/*
* switch mii access callbacks
*/
-@@ -2174,29 +2212,6 @@ static int bcm_enetsw_open(struct net_de
+@@ -2195,29 +2233,6 @@ static int bcm_enetsw_open(struct net_de
enetsw_writeb(priv, rgmii_ctrl, ENETSW_RGMII_CTRL_REG(i));
}
/* initialize flow control buffer allocation */
enet_dma_writel(priv, ENETDMA_BUFALLOC_FORCE_MASK | 0,
ENETDMA_BUFALLOC_REG(priv->rx_chan));
-@@ -2652,6 +2667,9 @@ static int bcm_enetsw_probe(struct platf
+@@ -2651,6 +2666,9 @@ static int bcm_enetsw_probe(struct platf
struct bcm63xx_enetsw_platform_data *pd;
struct resource *res_mem;
int ret, irq_rx, irq_tx;
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-@@ -2209,6 +2209,10 @@ static int bcm_enetsw_open(struct net_de
+@@ -2230,6 +2230,10 @@ static int bcm_enetsw_open(struct net_de
rgmii_ctrl = enetsw_readb(priv, ENETSW_RGMII_CTRL_REG(i));
rgmii_ctrl |= ENETSW_RGMII_CTRL_GMII_CLK_EN;
+++ /dev/null
-From 1606f78dc82e4f311ca5844a0605a62cd141a072 Mon Sep 17 00:00:00 2001
-From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
-Date: Mon, 30 Nov 2020 11:07:47 +0800
-Subject: [PATCH] bcm63xx: batch process RX path
-
-Use netif_receive_skb_list to batch process skb in RX.
-Tested on BCM6328 320 MHz using iperf3 -M 512, increasing performance
-by 12.5%.
-
-Before:
-[ ID] Interval Transfer Bandwidth Retr
-[ 4] 0.00-30.00 sec 120 MBytes 33.7 Mbits/sec 277 sender
-[ 4] 0.00-30.00 sec 120 MBytes 33.5 Mbits/sec receiver
-
-After:
-[ ID] Interval Transfer Bandwidth Retr
-[ 4] 0.00-30.00 sec 136 MBytes 37.9 Mbits/sec 203 sender
-[ 4] 0.00-30.00 sec 135 MBytes 37.7 Mbits/sec receiver
-
-Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
----
- drivers/net/ethernet/broadcom/bcm63xx_enet.c | 6 +++++-
- 1 file changed, 5 insertions(+), 1 deletion(-)
-
---- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-@@ -301,10 +301,12 @@ static int bcm_enet_receive_queue(struct
- struct bcm_enet_priv *priv;
- struct device *kdev;
- int processed;
-+ struct list_head rx_list;
-
- priv = netdev_priv(dev);
- kdev = &priv->pdev->dev;
- processed = 0;
-+ INIT_LIST_HEAD(&rx_list);
-
- /* don't scan ring further than number of refilled
- * descriptor */
-@@ -393,10 +395,12 @@ static int bcm_enet_receive_queue(struct
- skb->protocol = eth_type_trans(skb, dev);
- dev->stats.rx_packets++;
- dev->stats.rx_bytes += len;
-- netif_receive_skb(skb);
-+ list_add_tail(&skb->list, &rx_list);
-
- } while (--budget > 0);
-
-+ netif_receive_skb_list(&rx_list);
-+
- if (processed || !priv->rx_desc_count) {
- bcm_enet_refill_rx(dev);
-
+++ /dev/null
-From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
-From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
-Date: Mon, 30 Nov 2020 11:07:47 +0800
-Subject: [PATCH] bcm63xx: enetsw: switch to netdev_alloc_skb_ip_align
-
-Increases packet processing performance by 30%.
-Tested on BCM6328 320 MHz and iperf3 -M 512 for packet/sec performance.
-
-Before:
-[ ID] Interval Transfer Bandwidth Retr
-[ 4] 0.00-30.00 sec 120 MBytes 33.7 Mbits/sec 277 sender
-[ 4] 0.00-30.00 sec 120 MBytes 33.5 Mbits/sec receiver
-
-After:
-[ ID] Interval Transfer Bandwidth Retr
-[ 4] 0.00-30.00 sec 155 MBytes 43.3 Mbits/sec 354 sender
-[ 4] 0.00-30.00 sec 154 MBytes 43.1 Mbits/sec receiver
-
-Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
----
- drivers/net/ethernet/broadcom/bcm63xx_enet.c | 6 +++++-
- 1 file changed, 4 insertions(+), 1 deletion(-)
-
---- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-@@ -239,7 +239,10 @@ static int bcm_enet_refill_rx(struct net
- desc = &priv->rx_desc_cpu[desc_idx];
-
- if (!priv->rx_skb[desc_idx]) {
-- skb = netdev_alloc_skb(dev, priv->rx_skb_size);
-+ if (priv->enet_is_sw)
-+ skb = netdev_alloc_skb_ip_align(dev, priv->rx_skb_size);
-+ else
-+ skb = netdev_alloc_skb(dev, priv->rx_skb_size);
- if (!skb)
- break;
- priv->rx_skb[desc_idx] = skb;
+++ /dev/null
-From 3c0e8e1259d223a1493c47f7e277d05b93909b6b Mon Sep 17 00:00:00 2001
-From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
-Date: Mon, 2 Nov 2020 11:34:06 +0800
-Subject: [PATCH 1/2] bcm63xx: add BQL support
-
-Add Byte Queue Limits support to reduce/remove bufferbloat in bcm63xx target.
-
-Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
----
- bcm63xx_enet.c | 11 +++++++++++
- 1 file changed, 11 insertions(+)
-
---- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-@@ -423,9 +423,11 @@ static int bcm_enet_tx_reclaim(struct ne
- {
- struct bcm_enet_priv *priv;
- int released;
-+ unsigned int bytes;
-
- priv = netdev_priv(dev);
- released = 0;
-+ bytes = 0;
-
- while (priv->tx_desc_count < priv->tx_ring_size) {
- struct bcm_enet_desc *desc;
-@@ -461,10 +463,13 @@ static int bcm_enet_tx_reclaim(struct ne
- if (desc->len_stat & DMADESC_UNDER_MASK)
- dev->stats.tx_errors++;
-
-+ bytes += skb->len;
- dev_kfree_skb(skb);
- released++;
- }
-
-+ netdev_completed_queue(dev, released, bytes);
-+
- if (netif_queue_stopped(dev) && released)
- netif_wake_queue(dev);
-
-@@ -631,6 +636,8 @@ bcm_enet_start_xmit(struct sk_buff *skb,
- desc->len_stat = len_stat;
- wmb();
-
-+ netdev_sent_queue(dev, skb->len);
-+
- /* kick tx dma */
- enet_dmac_writel(priv, priv->dma_chan_en_mask,
- ENETDMAC_CHANCFG, priv->tx_chan);
-@@ -1051,6 +1058,8 @@ static int bcm_enet_open(struct net_devi
- else
- bcm_enet_adjust_link(dev);
-
-+ netdev_reset_queue(dev);
-+
- netif_start_queue(dev);
- return 0;
-
-@@ -2281,6 +2290,8 @@ static int bcm_enetsw_open(struct net_de
- enet_dmac_writel(priv, ENETDMAC_IR_PKTDONE_MASK,
- ENETDMAC_IRMASK, priv->tx_chan);
-
-+ netdev_reset_queue(dev);
-+
- netif_carrier_on(dev);
- netif_start_queue(dev);
-
+++ /dev/null
-From 79bfb73319098bc4cb701139a6677dcdec99182f Mon Sep 17 00:00:00 2001
-From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
-Date: Tue, 3 Nov 2020 08:14:35 +0800
-Subject: [PATCH 2/2] bcm63xx: support xmit_more in BQL
-
-Support bulking hardware TX queue by using xmit_more.
-
-Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
----
- bcm63xx_enet.c | 10 ++++++----
- 1 file changed, 6 insertions(+), 4 deletions(-)
-
---- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
-@@ -638,14 +638,16 @@ bcm_enet_start_xmit(struct sk_buff *skb,
-
- netdev_sent_queue(dev, skb->len);
-
-- /* kick tx dma */
-- enet_dmac_writel(priv, priv->dma_chan_en_mask,
-- ENETDMAC_CHANCFG, priv->tx_chan);
--
- /* stop queue if no more desc available */
- if (!priv->tx_desc_count)
- netif_stop_queue(dev);
-
-+ /* kick tx dma */
-+ if(!netdev_xmit_more() || !priv->tx_desc_count)
-+ enet_dmac_writel(priv, priv->dma_chan_en_mask,
-+ ENETDMAC_CHANCFG, priv->tx_chan);
-+
-+
- dev->stats.tx_bytes += skb->len;
- dev->stats.tx_packets++;
- ret = NETDEV_TX_OK;
-@@ -2713,7 +2715,7 @@ static int bcm_enetsw_probe(struct platf
- priv->irq_rx = irq_rx;
- priv->irq_tx = irq_tx;
- priv->rx_ring_size = BCMENET_DEF_RX_DESC;
-- priv->tx_ring_size = BCMENET_DEF_TX_DESC;
-+ priv->tx_ring_size = BCMENETSW_DEF_TX_DESC;
- priv->dma_maxburst = BCMENETSW_DMA_MAXBURST;
-
- pd = dev_get_platdata(&pdev->dev);
---- a/drivers/net/ethernet/broadcom/bcm63xx_enet.h
-+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
-@@ -15,6 +15,7 @@
- /* default number of descriptor */
- #define BCMENET_DEF_RX_DESC 64
- #define BCMENET_DEF_TX_DESC 32
-+#define BCMENETSW_DEF_TX_DESC 48
-
- /* maximum burst len for dma (4 bytes unit) */
- #define BCMENET_DMA_MAXBURST 16
projects / openwrt / staging / hauke.git / commitdiff