projects / openwrt / openwrt.git / blob
? search:


23e4a4dfcbf4122293bc0b5765243304ca9983f2
[openwrt/openwrt.git] /
1 From 155738a4f319538a09f734ce1f5a2eac3ada1de2 Mon Sep 17 00:00:00 2001
2 From: Lorenzo Bianconi <lorenzo@kernel.org>
3 Date: Wed, 27 Jul 2022 23:20:51 +0200
4 Subject: [PATCH] net: ethernet: mtk_eth_soc: introduce xdp multi-frag support
5
6 Add the capability to map non-linear xdp frames in XDP_TX and
7 ndo_xdp_xmit callback.
8
9 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
10 Signed-off-by: David S. Miller <davem@davemloft.net>
11 ---
12 drivers/net/ethernet/mediatek/mtk_eth_soc.c | 125 +++++++++++++-------
13 1 file changed, 82 insertions(+), 43 deletions(-)
14
15 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
16 +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
17 @@ -988,23 +988,22 @@ static void mtk_tx_unmap(struct mtk_eth
18 }
19 }
20
21 - if (tx_buf->type == MTK_TYPE_SKB) {
22 - if (tx_buf->data &&
23 - tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
24 + if (tx_buf->data && tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
25 + if (tx_buf->type == MTK_TYPE_SKB) {
26 struct sk_buff *skb = tx_buf->data;
27
28 if (napi)
29 napi_consume_skb(skb, napi);
30 else
31 dev_kfree_skb_any(skb);
32 - }
33 - } else if (tx_buf->data) {
34 - struct xdp_frame *xdpf = tx_buf->data;
35 + } else {
36 + struct xdp_frame *xdpf = tx_buf->data;
37
38 - if (napi && tx_buf->type == MTK_TYPE_XDP_TX)
39 - xdp_return_frame_rx_napi(xdpf);
40 - else
41 - xdp_return_frame(xdpf);
42 + if (napi && tx_buf->type == MTK_TYPE_XDP_TX)
43 + xdp_return_frame_rx_napi(xdpf);
44 + else
45 + xdp_return_frame(xdpf);
46 + }
47 }
48 tx_buf->flags = 0;
49 tx_buf->data = NULL;
50 @@ -1507,6 +1506,8 @@ static int mtk_xdp_frame_map(struct mtk_
51 mtk_tx_set_dma_desc(dev, txd, txd_info);
52
53 tx_buf->flags |= !mac->id ? MTK_TX_FLAGS_FPORT0 : MTK_TX_FLAGS_FPORT1;
54 + tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX;
55 + tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
56
57 txd_pdma = qdma_to_pdma(ring, txd);
58 setup_tx_buf(eth, tx_buf, txd_pdma, txd_info->addr, txd_info->size,
59 @@ -1518,43 +1519,69 @@ static int mtk_xdp_frame_map(struct mtk_
60 static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf,
61 struct net_device *dev, bool dma_map)
62 {
63 + struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
64 const struct mtk_soc_data *soc = eth->soc;
65 struct mtk_tx_ring *ring = &eth->tx_ring;
66 struct mtk_tx_dma_desc_info txd_info = {
67 .size = xdpf->len,
68 .first = true,
69 - .last = true,
70 + .last = !xdp_frame_has_frags(xdpf),
71 };
72 - int err = 0, index = 0, n_desc = 1;
73 - struct mtk_tx_dma *txd, *txd_pdma;
74 - struct mtk_tx_buf *tx_buf;
75 + int err, index = 0, n_desc = 1, nr_frags;
76 + struct mtk_tx_dma *htxd, *txd, *txd_pdma;
77 + struct mtk_tx_buf *htx_buf, *tx_buf;
78 + void *data = xdpf->data;
79
80 if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
81 return -EBUSY;
82
83 - if (unlikely(atomic_read(&ring->free_count) <= 1))
84 + nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
85 + if (unlikely(atomic_read(&ring->free_count) <= 1 + nr_frags))
86 return -EBUSY;
87
88 spin_lock(&eth->page_lock);
89
90 txd = ring->next_free;
91 if (txd == ring->last_free) {
92 - err = -ENOMEM;
93 - goto out;
94 + spin_unlock(&eth->page_lock);
95 + return -ENOMEM;
96 }
97 + htxd = txd;
98
99 tx_buf = mtk_desc_to_tx_buf(ring, txd, soc->txrx.txd_size);
100 memset(tx_buf, 0, sizeof(*tx_buf));
101 + htx_buf = tx_buf;
102
103 - err = mtk_xdp_frame_map(eth, dev, &txd_info, txd, tx_buf,
104 - xdpf->data, xdpf->headroom, index,
105 - dma_map);
106 - if (err < 0)
107 - goto out;
108 + for (;;) {
109 + err = mtk_xdp_frame_map(eth, dev, &txd_info, txd, tx_buf,
110 + data, xdpf->headroom, index, dma_map);
111 + if (err < 0)
112 + goto unmap;
113 +
114 + if (txd_info.last)
115 + break;
116
117 + if (MTK_HAS_CAPS(soc->caps, MTK_QDMA) || (index & 0x1)) {
118 + txd = mtk_qdma_phys_to_virt(ring, txd->txd2);
119 + txd_pdma = qdma_to_pdma(ring, txd);
120 + if (txd == ring->last_free)
121 + goto unmap;
122 +
123 + tx_buf = mtk_desc_to_tx_buf(ring, txd,
124 + soc->txrx.txd_size);
125 + memset(tx_buf, 0, sizeof(*tx_buf));
126 + n_desc++;
127 + }
128 +
129 + memset(&txd_info, 0, sizeof(struct mtk_tx_dma_desc_info));
130 + txd_info.size = skb_frag_size(&sinfo->frags[index]);
131 + txd_info.last = index + 1 == nr_frags;
132 + data = skb_frag_address(&sinfo->frags[index]);
133 +
134 + index++;
135 + }
136 /* store xdpf for cleanup */
137 - tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX;
138 - tx_buf->data = xdpf;
139 + htx_buf->data = xdpf;
140
141 if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
142 txd_pdma = qdma_to_pdma(ring, txd);
143 @@ -1581,7 +1608,24 @@ static int mtk_xdp_submit_frame(struct m
144 mtk_w32(eth, NEXT_DESP_IDX(idx, ring->dma_size),
145 MT7628_TX_CTX_IDX0);
146 }
147 -out:
148 +
149 + spin_unlock(&eth->page_lock);
150 +
151 + return 0;
152 +
153 +unmap:
154 + while (htxd != txd) {
155 + txd_pdma = qdma_to_pdma(ring, htxd);
156 + tx_buf = mtk_desc_to_tx_buf(ring, htxd, soc->txrx.txd_size);
157 + mtk_tx_unmap(eth, tx_buf, false);
158 +
159 + htxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
160 + if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA))
161 + txd_pdma->txd2 = TX_DMA_DESP2_DEF;
162 +
163 + htxd = mtk_qdma_phys_to_virt(ring, htxd->txd2);
164 + }
165 +
166 spin_unlock(&eth->page_lock);
167
168 return err;
169 @@ -1910,18 +1954,15 @@ static int mtk_poll_tx_qdma(struct mtk_e
170 if (!tx_buf->data)
171 break;
172
173 - if (tx_buf->type == MTK_TYPE_SKB &&
174 - tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
175 - struct sk_buff *skb = tx_buf->data;
176 + if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
177 + if (tx_buf->type == MTK_TYPE_SKB) {
178 + struct sk_buff *skb = tx_buf->data;
179
180 - bytes[mac] += skb->len;
181 - done[mac]++;
182 - budget--;
183 - } else if (tx_buf->type == MTK_TYPE_XDP_TX ||
184 - tx_buf->type == MTK_TYPE_XDP_NDO) {
185 + bytes[mac] += skb->len;
186 + done[mac]++;
187 + }
188 budget--;
189 }
190 -
191 mtk_tx_unmap(eth, tx_buf, true);
192
193 ring->last_free = desc;
194 @@ -1952,17 +1993,15 @@ static int mtk_poll_tx_pdma(struct mtk_e
195 if (!tx_buf->data)
196 break;
197
198 - if (tx_buf->type == MTK_TYPE_SKB &&
199 - tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
200 - struct sk_buff *skb = tx_buf->data;
201 - bytes[0] += skb->len;
202 - done[0]++;
203 - budget--;
204 - } else if (tx_buf->type == MTK_TYPE_XDP_TX ||
205 - tx_buf->type == MTK_TYPE_XDP_NDO) {
206 + if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
207 + if (tx_buf->type == MTK_TYPE_SKB) {
208 + struct sk_buff *skb = tx_buf->data;
209 +
210 + bytes[0] += skb->len;
211 + done[0]++;
212 + }
213 budget--;
214 }
215 -
216 mtk_tx_unmap(eth, tx_buf, true);
217
218 desc = ring->dma + cpu * eth->soc->txrx.txd_size;
OpenWrt Source Repository