projects / openwrt / staging / stintel.git / blob
? search:


c5501fb04023dac04aee9b28239a4bf730bb4d29
[openwrt/staging/stintel.git] /
1 From 23233e577ef973c2c5d0dd757a0a4605e34ecb57 Mon Sep 17 00:00:00 2001
2 From: Lorenzo Bianconi <lorenzo@kernel.org>
3 Date: Fri, 22 Jul 2022 09:19:36 +0200
4 Subject: [PATCH] net: ethernet: mtk_eth_soc: rely on page_pool for single page
5 buffers
6
7 Rely on page_pool allocator for single page buffers in order to keep
8 them dma mapped and add skb recycling support.
9
10 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
11 Signed-off-by: David S. Miller <davem@davemloft.net>
12 ---
13 drivers/net/ethernet/mediatek/Kconfig | 1 +
14 drivers/net/ethernet/mediatek/mtk_eth_soc.c | 185 +++++++++++++++-----
15 drivers/net/ethernet/mediatek/mtk_eth_soc.h | 10 ++
16 3 files changed, 156 insertions(+), 40 deletions(-)
17
18 --- a/drivers/net/ethernet/mediatek/Kconfig
19 +++ b/drivers/net/ethernet/mediatek/Kconfig
20 @@ -16,6 +16,7 @@ config NET_MEDIATEK_SOC
21 depends on NET_DSA || !NET_DSA
22 select PHYLINK
23 select DIMLIB
24 + select PAGE_POOL
25 help
26 This driver supports the gigabit ethernet MACs in the
27 MediaTek SoC family.
28 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
29 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
30 @@ -1389,6 +1389,68 @@ static void mtk_update_rx_cpu_idx(struct
31 }
32 }
33
34 +static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth,
35 + struct xdp_rxq_info *xdp_q,
36 + int id, int size)
37 +{
38 + struct page_pool_params pp_params = {
39 + .order = 0,
40 + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
41 + .pool_size = size,
42 + .nid = NUMA_NO_NODE,
43 + .dev = eth->dma_dev,
44 + .dma_dir = DMA_FROM_DEVICE,
45 + .offset = MTK_PP_HEADROOM,
46 + .max_len = MTK_PP_MAX_BUF_SIZE,
47 + };
48 + struct page_pool *pp;
49 + int err;
50 +
51 + pp = page_pool_create(&pp_params);
52 + if (IS_ERR(pp))
53 + return pp;
54 +
55 + err = xdp_rxq_info_reg(xdp_q, &eth->dummy_dev, id,
56 + eth->rx_napi.napi_id);
57 + if (err < 0)
58 + goto err_free_pp;
59 +
60 + err = xdp_rxq_info_reg_mem_model(xdp_q, MEM_TYPE_PAGE_POOL, pp);
61 + if (err)
62 + goto err_unregister_rxq;
63 +
64 + return pp;
65 +
66 +err_unregister_rxq:
67 + xdp_rxq_info_unreg(xdp_q);
68 +err_free_pp:
69 + page_pool_destroy(pp);
70 +
71 + return ERR_PTR(err);
72 +}
73 +
74 +static void *mtk_page_pool_get_buff(struct page_pool *pp, dma_addr_t *dma_addr,
75 + gfp_t gfp_mask)
76 +{
77 + struct page *page;
78 +
79 + page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN);
80 + if (!page)
81 + return NULL;
82 +
83 + *dma_addr = page_pool_get_dma_addr(page) + MTK_PP_HEADROOM;
84 + return page_address(page);
85 +}
86 +
87 +static void mtk_rx_put_buff(struct mtk_rx_ring *ring, void *data, bool napi)
88 +{
89 + if (ring->page_pool)
90 + page_pool_put_full_page(ring->page_pool,
91 + virt_to_head_page(data), napi);
92 + else
93 + skb_free_frag(data);
94 +}
95 +
96 static int mtk_poll_rx(struct napi_struct *napi, int budget,
97 struct mtk_eth *eth)
98 {
99 @@ -1402,9 +1464,9 @@ static int mtk_poll_rx(struct napi_struc
100
101 while (done < budget) {
102 unsigned int pktlen, *rxdcsum;
103 + u32 hash, reason, reserve_len;
104 struct net_device *netdev;
105 dma_addr_t dma_addr;
106 - u32 hash, reason;
107 int mac = 0;
108
109 ring = mtk_get_rx_ring(eth);
110 @@ -1435,36 +1497,54 @@ static int mtk_poll_rx(struct napi_struc
111 goto release_desc;
112
113 /* alloc new buffer */
114 - if (ring->frag_size <= PAGE_SIZE)
115 - new_data = napi_alloc_frag(ring->frag_size);
116 - else
117 - new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
118 - if (unlikely(!new_data)) {
119 - netdev->stats.rx_dropped++;
120 - goto release_desc;
121 - }
122 - dma_addr = dma_map_single(eth->dma_dev,
123 - new_data + NET_SKB_PAD +
124 - eth->ip_align,
125 - ring->buf_size,
126 - DMA_FROM_DEVICE);
127 - if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr))) {
128 - skb_free_frag(new_data);
129 - netdev->stats.rx_dropped++;
130 - goto release_desc;
131 - }
132 + if (ring->page_pool) {
133 + new_data = mtk_page_pool_get_buff(ring->page_pool,
134 + &dma_addr,
135 + GFP_ATOMIC);
136 + if (unlikely(!new_data)) {
137 + netdev->stats.rx_dropped++;
138 + goto release_desc;
139 + }
140 + } else {
141 + if (ring->frag_size <= PAGE_SIZE)
142 + new_data = napi_alloc_frag(ring->frag_size);
143 + else
144 + new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
145 +
146 + if (unlikely(!new_data)) {
147 + netdev->stats.rx_dropped++;
148 + goto release_desc;
149 + }
150
151 - dma_unmap_single(eth->dma_dev, trxd.rxd1,
152 - ring->buf_size, DMA_FROM_DEVICE);
153 + dma_addr = dma_map_single(eth->dma_dev,
154 + new_data + NET_SKB_PAD + eth->ip_align,
155 + ring->buf_size, DMA_FROM_DEVICE);
156 + if (unlikely(dma_mapping_error(eth->dma_dev,
157 + dma_addr))) {
158 + skb_free_frag(new_data);
159 + netdev->stats.rx_dropped++;
160 + goto release_desc;
161 + }
162 +
163 + dma_unmap_single(eth->dma_dev, trxd.rxd1,
164 + ring->buf_size, DMA_FROM_DEVICE);
165 + }
166
167 /* receive data */
168 skb = build_skb(data, ring->frag_size);
169 if (unlikely(!skb)) {
170 - skb_free_frag(data);
171 + mtk_rx_put_buff(ring, data, true);
172 netdev->stats.rx_dropped++;
173 goto skip_rx;
174 }
175 - skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
176 +
177 + if (ring->page_pool) {
178 + reserve_len = MTK_PP_HEADROOM;
179 + skb_mark_for_recycle(skb);
180 + } else {
181 + reserve_len = NET_SKB_PAD + NET_IP_ALIGN;
182 + }
183 + skb_reserve(skb, reserve_len);
184
185 pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
186 skb->dev = netdev;
187 @@ -1518,7 +1598,6 @@ static int mtk_poll_rx(struct napi_struc
188 skip_rx:
189 ring->data[idx] = new_data;
190 rxd->rxd1 = (unsigned int)dma_addr;
191 -
192 release_desc:
193 if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
194 rxd->rxd2 = RX_DMA_LSO;
195 @@ -1526,7 +1605,6 @@ release_desc:
196 rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size);
197
198 ring->calc_idx = idx;
199 -
200 done++;
201 }
202
203 @@ -1890,13 +1968,15 @@ static int mtk_rx_alloc(struct mtk_eth *
204 if (!ring->data)
205 return -ENOMEM;
206
207 - for (i = 0; i < rx_dma_size; i++) {
208 - if (ring->frag_size <= PAGE_SIZE)
209 - ring->data[i] = netdev_alloc_frag(ring->frag_size);
210 - else
211 - ring->data[i] = mtk_max_lro_buf_alloc(GFP_KERNEL);
212 - if (!ring->data[i])
213 - return -ENOMEM;
214 + if (!eth->hwlro) {
215 + struct page_pool *pp;
216 +
217 + pp = mtk_create_page_pool(eth, &ring->xdp_q, ring_no,
218 + rx_dma_size);
219 + if (IS_ERR(pp))
220 + return PTR_ERR(pp);
221 +
222 + ring->page_pool = pp;
223 }
224
225 ring->dma = dma_alloc_coherent(eth->dma_dev,
226 @@ -1907,16 +1987,33 @@ static int mtk_rx_alloc(struct mtk_eth *
227
228 for (i = 0; i < rx_dma_size; i++) {
229 struct mtk_rx_dma_v2 *rxd;
230 -
231 - dma_addr_t dma_addr = dma_map_single(eth->dma_dev,
232 - ring->data[i] + NET_SKB_PAD + eth->ip_align,
233 - ring->buf_size,
234 - DMA_FROM_DEVICE);
235 - if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr)))
236 - return -ENOMEM;
237 + dma_addr_t dma_addr;
238 + void *data;
239
240 rxd = ring->dma + i * eth->soc->txrx.rxd_size;
241 + if (ring->page_pool) {
242 + data = mtk_page_pool_get_buff(ring->page_pool,
243 + &dma_addr, GFP_KERNEL);
244 + if (!data)
245 + return -ENOMEM;
246 + } else {
247 + if (ring->frag_size <= PAGE_SIZE)
248 + data = netdev_alloc_frag(ring->frag_size);
249 + else
250 + data = mtk_max_lro_buf_alloc(GFP_KERNEL);
251 +
252 + if (!data)
253 + return -ENOMEM;
254 +
255 + dma_addr = dma_map_single(eth->dma_dev,
256 + data + NET_SKB_PAD + eth->ip_align,
257 + ring->buf_size, DMA_FROM_DEVICE);
258 + if (unlikely(dma_mapping_error(eth->dma_dev,
259 + dma_addr)))
260 + return -ENOMEM;
261 + }
262 rxd->rxd1 = (unsigned int)dma_addr;
263 + ring->data[i] = data;
264
265 if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
266 rxd->rxd2 = RX_DMA_LSO;
267 @@ -1932,6 +2029,7 @@ static int mtk_rx_alloc(struct mtk_eth *
268 rxd->rxd8 = 0;
269 }
270 }
271 +
272 ring->dma_size = rx_dma_size;
273 ring->calc_idx_update = false;
274 ring->calc_idx = rx_dma_size - 1;
275 @@ -1983,7 +2081,7 @@ static void mtk_rx_clean(struct mtk_eth
276
277 dma_unmap_single(eth->dma_dev, rxd->rxd1,
278 ring->buf_size, DMA_FROM_DEVICE);
279 - skb_free_frag(ring->data[i]);
280 + mtk_rx_put_buff(ring, ring->data[i], false);
281 }
282 kfree(ring->data);
283 ring->data = NULL;
284 @@ -1995,6 +2093,13 @@ static void mtk_rx_clean(struct mtk_eth
285 ring->dma, ring->phys);
286 ring->dma = NULL;
287 }
288 +
289 + if (ring->page_pool) {
290 + if (xdp_rxq_info_is_reg(&ring->xdp_q))
291 + xdp_rxq_info_unreg(&ring->xdp_q);
292 + page_pool_destroy(ring->page_pool);
293 + ring->page_pool = NULL;
294 + }
295 }
296
297 static int mtk_hwlro_rx_init(struct mtk_eth *eth)
298 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
299 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
300 @@ -18,6 +18,8 @@
301 #include <linux/rhashtable.h>
302 #include <linux/dim.h>
303 #include <linux/bitfield.h>
304 +#include <net/page_pool.h>
305 +#include <linux/bpf_trace.h>
306 #include "mtk_ppe.h"
307
308 #define MTK_QDMA_PAGE_SIZE 2048
309 @@ -49,6 +51,11 @@
310 #define MTK_HW_FEATURES_MT7628 (NETIF_F_SG | NETIF_F_RXCSUM)
311 #define NEXT_DESP_IDX(X, Y) (((X) + 1) & ((Y) - 1))
312
313 +#define MTK_PP_HEADROOM XDP_PACKET_HEADROOM
314 +#define MTK_PP_PAD (MTK_PP_HEADROOM + \
315 + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
316 +#define MTK_PP_MAX_BUF_SIZE (PAGE_SIZE - MTK_PP_PAD)
317 +
318 #define MTK_QRX_OFFSET 0x10
319
320 #define MTK_MAX_RX_RING_NUM 4
321 @@ -743,6 +750,9 @@ struct mtk_rx_ring {
322 bool calc_idx_update;
323 u16 calc_idx;
324 u32 crx_idx_reg;
325 + /* page_pool */
326 + struct page_pool *page_pool;
327 + struct xdp_rxq_info xdp_q;
328 };
329
330 enum mkt_eth_capabilities {
Staging tree of Stijn Tintel