From dd2bc8e9c0685d8eaaaf06e65919e31d60478411 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 15 Nov 2011 07:30:05 +0000 Subject: [PATCH] bnx2: switch to build_skb() infrastructure This is very similar to bnx2x conversion, but bnx2 only requires 16bytes alignement at start of the received frame to store its l2_fhdr, so goal was not to reduce skb truesize (in fact it should not change after this patch) Using build_skb() reduces cache line misses in the driver, since we use cache hot skb instead of cold ones. Number of in-flight sk_buff structures is lower, they are more likely recycled in SLUB caches while still hot. Signed-off-by: Eric Dumazet CC: Michael Chan CC: Eilon Greenstein Reviewed-by: Michael Chan Signed-off-by: David S. Miller --- drivers/net/ethernet/broadcom/bnx2.c | 137 ++++++++++++++------------- drivers/net/ethernet/broadcom/bnx2.h | 17 +++- 2 files changed, 85 insertions(+), 69 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c index 6b7cd1e80ada..66f6e7f654c3 100644 --- a/drivers/net/ethernet/broadcom/bnx2.c +++ b/drivers/net/ethernet/broadcom/bnx2.c @@ -2725,31 +2725,27 @@ bnx2_free_rx_page(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, u16 index) } static inline int -bnx2_alloc_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, u16 index, gfp_t gfp) +bnx2_alloc_rx_data(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, u16 index, gfp_t gfp) { - struct sk_buff *skb; + u8 *data; struct sw_bd *rx_buf = &rxr->rx_buf_ring[index]; dma_addr_t mapping; struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(index)][RX_IDX(index)]; - unsigned long align; - skb = __netdev_alloc_skb(bp->dev, bp->rx_buf_size, gfp); - if (skb == NULL) { + data = kmalloc(bp->rx_buf_size, gfp); + if (!data) return -ENOMEM; - } - if (unlikely((align = (unsigned long) skb->data & (BNX2_RX_ALIGN - 1)))) - skb_reserve(skb, BNX2_RX_ALIGN - align); - - mapping = dma_map_single(&bp->pdev->dev, skb->data, bp->rx_buf_use_size, + mapping = dma_map_single(&bp->pdev->dev, + get_l2_fhdr(data), + bp->rx_buf_use_size, PCI_DMA_FROMDEVICE); if (dma_mapping_error(&bp->pdev->dev, mapping)) { - dev_kfree_skb(skb); + kfree(data); return -EIO; } - rx_buf->skb = skb; - rx_buf->desc = (struct l2_fhdr *) skb->data; + rx_buf->data = data; dma_unmap_addr_set(rx_buf, mapping, mapping); rxbd->rx_bd_haddr_hi = (u64) mapping >> 32; @@ -2956,8 +2952,8 @@ bnx2_reuse_rx_skb_pages(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, } static inline void -bnx2_reuse_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, - struct sk_buff *skb, u16 cons, u16 prod) +bnx2_reuse_rx_data(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, + u8 *data, u16 cons, u16 prod) { struct sw_bd *cons_rx_buf, *prod_rx_buf; struct rx_bd *cons_bd, *prod_bd; @@ -2971,8 +2967,7 @@ bnx2_reuse_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, rxr->rx_prod_bseq += bp->rx_buf_use_size; - prod_rx_buf->skb = skb; - prod_rx_buf->desc = (struct l2_fhdr *) skb->data; + prod_rx_buf->data = data; if (cons == prod) return; @@ -2986,33 +2981,39 @@ bnx2_reuse_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, prod_bd->rx_bd_haddr_lo = cons_bd->rx_bd_haddr_lo; } -static int -bnx2_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, struct sk_buff *skb, +static struct sk_buff * +bnx2_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, u8 *data, unsigned int len, unsigned int hdr_len, dma_addr_t dma_addr, u32 ring_idx) { int err; u16 prod = ring_idx & 0xffff; + struct sk_buff *skb; - err = bnx2_alloc_rx_skb(bp, rxr, prod, GFP_ATOMIC); + err = bnx2_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); if (unlikely(err)) { - bnx2_reuse_rx_skb(bp, rxr, skb, (u16) (ring_idx >> 16), prod); + bnx2_reuse_rx_data(bp, rxr, data, (u16) (ring_idx >> 16), prod); +error: if (hdr_len) { unsigned int raw_len = len + 4; int pages = PAGE_ALIGN(raw_len - hdr_len) >> PAGE_SHIFT; bnx2_reuse_rx_skb_pages(bp, rxr, NULL, pages); } - return err; + return NULL; } - skb_reserve(skb, BNX2_RX_OFFSET); dma_unmap_single(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, PCI_DMA_FROMDEVICE); - + skb = build_skb(data); + if (!skb) { + kfree(data); + goto error; + } + skb_reserve(skb, ((u8 *)get_l2_fhdr(data) - data) + BNX2_RX_OFFSET); if (hdr_len == 0) { skb_put(skb, len); - return 0; + return skb; } else { unsigned int i, frag_len, frag_size, pages; struct sw_pg *rx_pg; @@ -3043,7 +3044,7 @@ bnx2_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, struct sk_buff *skb, skb_frag_size_sub(frag, tail); skb->data_len -= tail; } - return 0; + return skb; } rx_pg = &rxr->rx_pg_ring[pg_cons]; @@ -3065,7 +3066,7 @@ bnx2_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, struct sk_buff *skb, rxr->rx_pg_prod = pg_prod; bnx2_reuse_rx_skb_pages(bp, rxr, skb, pages - i); - return err; + return NULL; } dma_unmap_page(&bp->pdev->dev, mapping_old, @@ -3082,7 +3083,7 @@ bnx2_rx_skb(struct bnx2 *bp, struct bnx2_rx_ring_info *rxr, struct sk_buff *skb, rxr->rx_pg_prod = pg_prod; rxr->rx_pg_cons = pg_cons; } - return 0; + return skb; } static inline u16 @@ -3121,19 +3122,17 @@ bnx2_rx_int(struct bnx2 *bp, struct bnx2_napi *bnapi, int budget) struct sw_bd *rx_buf, *next_rx_buf; struct sk_buff *skb; dma_addr_t dma_addr; + u8 *data; sw_ring_cons = RX_RING_IDX(sw_cons); sw_ring_prod = RX_RING_IDX(sw_prod); rx_buf = &rxr->rx_buf_ring[sw_ring_cons]; - skb = rx_buf->skb; - prefetchw(skb); + data = rx_buf->data; + rx_buf->data = NULL; - next_rx_buf = - &rxr->rx_buf_ring[RX_RING_IDX(NEXT_RX_BD(sw_cons))]; - prefetch(next_rx_buf->desc); - - rx_buf->skb = NULL; + rx_hdr = get_l2_fhdr(data); + prefetch(rx_hdr); dma_addr = dma_unmap_addr(rx_buf, mapping); @@ -3141,7 +3140,10 @@ bnx2_rx_int(struct bnx2 *bp, struct bnx2_napi *bnapi, int budget) BNX2_RX_OFFSET + BNX2_RX_COPY_THRESH, PCI_DMA_FROMDEVICE); - rx_hdr = rx_buf->desc; + next_rx_buf = + &rxr->rx_buf_ring[RX_RING_IDX(NEXT_RX_BD(sw_cons))]; + prefetch(get_l2_fhdr(next_rx_buf->data)); + len = rx_hdr->l2_fhdr_pkt_len; status = rx_hdr->l2_fhdr_status; @@ -3160,7 +3162,7 @@ bnx2_rx_int(struct bnx2 *bp, struct bnx2_napi *bnapi, int budget) L2_FHDR_ERRORS_TOO_SHORT | L2_FHDR_ERRORS_GIANT_FRAME))) { - bnx2_reuse_rx_skb(bp, rxr, skb, sw_ring_cons, + bnx2_reuse_rx_data(bp, rxr, data, sw_ring_cons, sw_ring_prod); if (pg_ring_used) { int pages; @@ -3175,30 +3177,29 @@ bnx2_rx_int(struct bnx2 *bp, struct bnx2_napi *bnapi, int budget) len -= 4; if (len <= bp->rx_copy_thresh) { - struct sk_buff *new_skb; - - new_skb = netdev_alloc_skb(bp->dev, len + 6); - if (new_skb == NULL) { - bnx2_reuse_rx_skb(bp, rxr, skb, sw_ring_cons, + skb = netdev_alloc_skb(bp->dev, len + 6); + if (skb == NULL) { + bnx2_reuse_rx_data(bp, rxr, data, sw_ring_cons, sw_ring_prod); goto next_rx; } /* aligned copy */ - skb_copy_from_linear_data_offset(skb, - BNX2_RX_OFFSET - 6, - new_skb->data, len + 6); - skb_reserve(new_skb, 6); - skb_put(new_skb, len); + memcpy(skb->data, + (u8 *)rx_hdr + BNX2_RX_OFFSET - 6, + len + 6); + skb_reserve(skb, 6); + skb_put(skb, len); - bnx2_reuse_rx_skb(bp, rxr, skb, + bnx2_reuse_rx_data(bp, rxr, data, sw_ring_cons, sw_ring_prod); - skb = new_skb; - } else if (unlikely(bnx2_rx_skb(bp, rxr, skb, len, hdr_len, - dma_addr, (sw_ring_cons << 16) | sw_ring_prod))) - goto next_rx; - + } else { + skb = bnx2_rx_skb(bp, rxr, data, len, hdr_len, dma_addr, + (sw_ring_cons << 16) | sw_ring_prod); + if (!skb) + goto next_rx; + } if ((status & L2_FHDR_STATUS_L2_VLAN_TAG) && !(bp->rx_mode & BNX2_EMAC_RX_MODE_KEEP_VLAN_TAG)) __vlan_hwaccel_put_tag(skb, rx_hdr->l2_fhdr_vlan_tag); @@ -5225,7 +5226,7 @@ bnx2_init_rx_ring(struct bnx2 *bp, int ring_num) ring_prod = prod = rxr->rx_prod; for (i = 0; i < bp->rx_ring_size; i++) { - if (bnx2_alloc_rx_skb(bp, rxr, ring_prod, GFP_KERNEL) < 0) { + if (bnx2_alloc_rx_data(bp, rxr, ring_prod, GFP_KERNEL) < 0) { netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d skbs only\n", ring_num, i, bp->rx_ring_size); break; @@ -5320,7 +5321,7 @@ bnx2_set_rx_ring_size(struct bnx2 *bp, u32 size) rx_size = bp->dev->mtu + ETH_HLEN + BNX2_RX_OFFSET + 8; rx_space = SKB_DATA_ALIGN(rx_size + BNX2_RX_ALIGN) + NET_SKB_PAD + - sizeof(struct skb_shared_info); + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); bp->rx_copy_thresh = BNX2_RX_COPY_THRESH; bp->rx_pg_ring_size = 0; @@ -5342,8 +5343,9 @@ bnx2_set_rx_ring_size(struct bnx2 *bp, u32 size) } bp->rx_buf_use_size = rx_size; - /* hw alignment */ - bp->rx_buf_size = bp->rx_buf_use_size + BNX2_RX_ALIGN; + /* hw alignment + build_skb() overhead*/ + bp->rx_buf_size = SKB_DATA_ALIGN(bp->rx_buf_use_size + BNX2_RX_ALIGN) + + NET_SKB_PAD + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); bp->rx_jumbo_thresh = rx_size - BNX2_RX_OFFSET; bp->rx_ring_size = size; bp->rx_max_ring = bnx2_find_max_ring(size, MAX_RX_RINGS); @@ -5409,9 +5411,9 @@ bnx2_free_rx_skbs(struct bnx2 *bp) for (j = 0; j < bp->rx_max_ring_idx; j++) { struct sw_bd *rx_buf = &rxr->rx_buf_ring[j]; - struct sk_buff *skb = rx_buf->skb; + u8 *data = rx_buf->data; - if (skb == NULL) + if (data == NULL) continue; dma_unmap_single(&bp->pdev->dev, @@ -5419,9 +5421,9 @@ bnx2_free_rx_skbs(struct bnx2 *bp) bp->rx_buf_use_size, PCI_DMA_FROMDEVICE); - rx_buf->skb = NULL; + rx_buf->data = NULL; - dev_kfree_skb(skb); + kfree(data); } for (j = 0; j < bp->rx_max_pg_ring_idx; j++) bnx2_free_rx_page(bp, rxr, j); @@ -5727,7 +5729,8 @@ static int bnx2_run_loopback(struct bnx2 *bp, int loopback_mode) { unsigned int pkt_size, num_pkts, i; - struct sk_buff *skb, *rx_skb; + struct sk_buff *skb; + u8 *data; unsigned char *packet; u16 rx_start_idx, rx_idx; dma_addr_t map; @@ -5819,14 +5822,14 @@ bnx2_run_loopback(struct bnx2 *bp, int loopback_mode) } rx_buf = &rxr->rx_buf_ring[rx_start_idx]; - rx_skb = rx_buf->skb; + data = rx_buf->data; - rx_hdr = rx_buf->desc; - skb_reserve(rx_skb, BNX2_RX_OFFSET); + rx_hdr = get_l2_fhdr(data); + data = (u8 *)rx_hdr + BNX2_RX_OFFSET; dma_sync_single_for_cpu(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping), - bp->rx_buf_size, PCI_DMA_FROMDEVICE); + bp->rx_buf_use_size, PCI_DMA_FROMDEVICE); if (rx_hdr->l2_fhdr_status & (L2_FHDR_ERRORS_BAD_CRC | @@ -5843,7 +5846,7 @@ bnx2_run_loopback(struct bnx2 *bp, int loopback_mode) } for (i = 14; i < pkt_size; i++) { - if (*(rx_skb->data + i) != (unsigned char) (i & 0xff)) { + if (*(data + i) != (unsigned char) (i & 0xff)) { goto loopback_test_done; } } diff --git a/drivers/net/ethernet/broadcom/bnx2.h b/drivers/net/ethernet/broadcom/bnx2.h index 99d31a7d6aaa..1db2d51ba3f1 100644 --- a/drivers/net/ethernet/broadcom/bnx2.h +++ b/drivers/net/ethernet/broadcom/bnx2.h @@ -6563,12 +6563,25 @@ struct l2_fhdr { #define MB_TX_CID_ADDR MB_GET_CID_ADDR(TX_CID) #define MB_RX_CID_ADDR MB_GET_CID_ADDR(RX_CID) +/* + * This driver uses new build_skb() API : + * RX ring buffer contains pointer to kmalloc() data only, + * skb are built only after Hardware filled the frame. + */ struct sw_bd { - struct sk_buff *skb; - struct l2_fhdr *desc; + u8 *data; DEFINE_DMA_UNMAP_ADDR(mapping); }; +/* Its faster to compute this from data than storing it in sw_bd + * (less cache misses) + */ +static inline struct l2_fhdr *get_l2_fhdr(u8 *data) +{ + return (struct l2_fhdr *)(PTR_ALIGN(data, BNX2_RX_ALIGN) + NET_SKB_PAD); +} + + struct sw_pg { struct page *page; DEFINE_DMA_UNMAP_ADDR(mapping); -- 2.30.2