1 From d27de0ef5ef995df2cc5f5c006c0efcf0a62b6af Mon Sep 17 00:00:00 2001
2 From: Sieng Piaw Liew <liew.s.piaw@gmail.com>
3 Date: Wed, 6 Jan 2021 22:42:07 +0800
4 Subject: [PATCH 6/7] bcm63xx_enet: convert to build_skb
6 We can increase the efficiency of rx path by using buffers to receive
7 packets then build SKBs around them just before passing into the network
8 stack. In contrast, preallocating SKBs too early reduces CPU cache
11 Check if we're in NAPI context when refilling RX. Normally we're almost
12 always running in NAPI context. Dispatch to napi_alloc_frag directly
13 instead of relying on netdev_alloc_frag which does the same but
14 with the overhead of local_bh_disable/enable.
16 Tested on BCM6328 320 MHz and iperf3 -M 512 to measure packet/sec
17 performance. Included netif_receive_skb_list and NET_IP_ALIGN
21 [ ID] Interval Transfer Bandwidth Retr
22 [ 4] 0.00-10.00 sec 49.9 MBytes 41.9 Mbits/sec 197 sender
23 [ 4] 0.00-10.00 sec 49.3 MBytes 41.3 Mbits/sec receiver
26 [ ID] Interval Transfer Bandwidth Retr
27 [ 4] 0.00-30.00 sec 171 MBytes 47.8 Mbits/sec 272 sender
28 [ 4] 0.00-30.00 sec 170 MBytes 47.6 Mbits/sec receiver
30 Signed-off-by: Sieng Piaw Liew <liew.s.piaw@gmail.com>
31 Acked-by: Florian Fainelli <f.fainelli@gmail.com>
32 Signed-off-by: Jakub Kicinski <kuba@kernel.org>
34 drivers/net/ethernet/broadcom/bcm63xx_enet.c | 111 ++++++++++---------
35 drivers/net/ethernet/broadcom/bcm63xx_enet.h | 14 ++-
36 2 files changed, 71 insertions(+), 54 deletions(-)
38 --- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
39 +++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
40 @@ -220,7 +220,7 @@ static void bcm_enet_mdio_write_mii(stru
44 -static int bcm_enet_refill_rx(struct net_device *dev)
45 +static int bcm_enet_refill_rx(struct net_device *dev, bool napi_mode)
47 struct bcm_enet_priv *priv;
49 @@ -228,29 +228,29 @@ static int bcm_enet_refill_rx(struct net
51 while (priv->rx_desc_count < priv->rx_ring_size) {
52 struct bcm_enet_desc *desc;
53 - struct sk_buff *skb;
58 desc_idx = priv->rx_dirty_desc;
59 desc = &priv->rx_desc_cpu[desc_idx];
61 - if (!priv->rx_skb[desc_idx]) {
62 - if (priv->enet_is_sw)
63 - skb = netdev_alloc_skb_ip_align(dev, priv->rx_skb_size);
64 + if (!priv->rx_buf[desc_idx]) {
67 + if (likely(napi_mode))
68 + buf = napi_alloc_frag(priv->rx_frag_size);
70 - skb = netdev_alloc_skb(dev, priv->rx_skb_size);
72 + buf = netdev_alloc_frag(priv->rx_frag_size);
75 - priv->rx_skb[desc_idx] = skb;
76 - p = dma_map_single(&priv->pdev->dev, skb->data,
80 + priv->rx_buf[desc_idx] = buf;
81 + desc->address = dma_map_single(&priv->pdev->dev,
82 + buf + priv->rx_buf_offset,
87 - len_stat = priv->rx_skb_size << DMADESC_LENGTH_SHIFT;
88 + len_stat = priv->rx_buf_size << DMADESC_LENGTH_SHIFT;
89 len_stat |= DMADESC_OWNER_MASK;
90 if (priv->rx_dirty_desc == priv->rx_ring_size - 1) {
91 len_stat |= (DMADESC_WRAP_MASK >> priv->dma_desc_shift);
92 @@ -290,7 +290,7 @@ static void bcm_enet_refill_rx_timer(str
93 struct net_device *dev = priv->net_dev;
95 spin_lock(&priv->rx_lock);
96 - bcm_enet_refill_rx(dev);
97 + bcm_enet_refill_rx(dev, false);
98 spin_unlock(&priv->rx_lock);
101 @@ -320,6 +320,7 @@ static int bcm_enet_receive_queue(struct
107 desc_idx = priv->rx_curr_desc;
108 desc = &priv->rx_desc_cpu[desc_idx];
109 @@ -365,16 +366,14 @@ static int bcm_enet_receive_queue(struct
113 - skb = priv->rx_skb[desc_idx];
114 + buf = priv->rx_buf[desc_idx];
115 len = (len_stat & DMADESC_LENGTH_MASK) >> DMADESC_LENGTH_SHIFT;
116 /* don't include FCS */
119 if (len < copybreak) {
120 - struct sk_buff *nskb;
122 - nskb = napi_alloc_skb(&priv->napi, len);
124 + skb = napi_alloc_skb(&priv->napi, len);
125 + if (unlikely(!skb)) {
126 /* forget packet, just rearm desc */
127 dev->stats.rx_dropped++;
129 @@ -382,14 +381,21 @@ static int bcm_enet_receive_queue(struct
131 dma_sync_single_for_cpu(kdev, desc->address,
132 len, DMA_FROM_DEVICE);
133 - memcpy(nskb->data, skb->data, len);
134 + memcpy(skb->data, buf + priv->rx_buf_offset, len);
135 dma_sync_single_for_device(kdev, desc->address,
136 len, DMA_FROM_DEVICE);
139 - dma_unmap_single(&priv->pdev->dev, desc->address,
140 - priv->rx_skb_size, DMA_FROM_DEVICE);
141 - priv->rx_skb[desc_idx] = NULL;
142 + dma_unmap_single(kdev, desc->address,
143 + priv->rx_buf_size, DMA_FROM_DEVICE);
144 + priv->rx_buf[desc_idx] = NULL;
146 + skb = build_skb(buf, priv->rx_frag_size);
147 + if (unlikely(!skb)) {
148 + skb_free_frag(buf);
149 + dev->stats.rx_dropped++;
152 + skb_reserve(skb, priv->rx_buf_offset);
156 @@ -403,7 +409,7 @@ static int bcm_enet_receive_queue(struct
157 netif_receive_skb_list(&rx_list);
159 if (processed || !priv->rx_desc_count) {
160 - bcm_enet_refill_rx(dev);
161 + bcm_enet_refill_rx(dev, true);
164 enet_dmac_writel(priv, priv->dma_chan_en_mask,
165 @@ -860,22 +866,22 @@ static void bcm_enet_adjust_link(struct
166 priv->pause_tx ? "tx" : "off");
169 -static void bcm_enet_free_rx_skb_ring(struct device *kdev, struct bcm_enet_priv *priv)
170 +static void bcm_enet_free_rx_buf_ring(struct device *kdev, struct bcm_enet_priv *priv)
174 for (i = 0; i < priv->rx_ring_size; i++) {
175 struct bcm_enet_desc *desc;
177 - if (!priv->rx_skb[i])
178 + if (!priv->rx_buf[i])
181 desc = &priv->rx_desc_cpu[i];
182 - dma_unmap_single(kdev, desc->address, priv->rx_skb_size,
183 + dma_unmap_single(kdev, desc->address, priv->rx_buf_size,
185 - kfree_skb(priv->rx_skb[i]);
186 + skb_free_frag(priv->rx_buf[i]);
188 - kfree(priv->rx_skb);
189 + kfree(priv->rx_buf);
193 @@ -987,10 +993,10 @@ static int bcm_enet_open(struct net_devi
194 priv->tx_curr_desc = 0;
195 spin_lock_init(&priv->tx_lock);
197 - /* init & fill rx ring with skbs */
198 - priv->rx_skb = kcalloc(priv->rx_ring_size, sizeof(struct sk_buff *),
199 + /* init & fill rx ring with buffers */
200 + priv->rx_buf = kcalloc(priv->rx_ring_size, sizeof(void *),
202 - if (!priv->rx_skb) {
203 + if (!priv->rx_buf) {
205 goto out_free_tx_skb;
207 @@ -1007,8 +1013,8 @@ static int bcm_enet_open(struct net_devi
208 enet_dmac_writel(priv, ENETDMA_BUFALLOC_FORCE_MASK | 0,
209 ENETDMAC_BUFALLOC, priv->rx_chan);
211 - if (bcm_enet_refill_rx(dev)) {
212 - dev_err(kdev, "cannot allocate rx skb queue\n");
213 + if (bcm_enet_refill_rx(dev, false)) {
214 + dev_err(kdev, "cannot allocate rx buffer queue\n");
218 @@ -1102,7 +1108,7 @@ static int bcm_enet_open(struct net_devi
222 - bcm_enet_free_rx_skb_ring(kdev, priv);
223 + bcm_enet_free_rx_buf_ring(kdev, priv);
227 @@ -1208,8 +1214,8 @@ static int bcm_enet_stop(struct net_devi
228 /* force reclaim of all tx buffers */
229 bcm_enet_tx_reclaim(dev, 1);
231 - /* free the rx skb ring */
232 - bcm_enet_free_rx_skb_ring(kdev, priv);
233 + /* free the rx buffer ring */
234 + bcm_enet_free_rx_buf_ring(kdev, priv);
236 /* free remaining allocated memory */
238 @@ -1633,9 +1639,12 @@ static int bcm_enet_change_mtu(struct ne
239 * align rx buffer size to dma burst len, account FCS since
242 - priv->rx_skb_size = ALIGN(actual_mtu + ETH_FCS_LEN,
243 + priv->rx_buf_size = ALIGN(actual_mtu + ETH_FCS_LEN,
244 priv->dma_maxburst * 4);
246 + priv->rx_frag_size = SKB_DATA_ALIGN(priv->rx_buf_offset + priv->rx_buf_size) +
247 + SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
252 @@ -1720,6 +1729,7 @@ static int bcm_enet_probe(struct platfor
254 priv->enet_is_sw = false;
255 priv->dma_maxburst = BCMENET_DMA_MAXBURST;
256 + priv->rx_buf_offset = NET_SKB_PAD;
258 ret = bcm_enet_change_mtu(dev, dev->mtu);
260 @@ -2137,7 +2147,7 @@ static int bcm_enetsw_open(struct net_de
261 priv->tx_skb = kcalloc(priv->tx_ring_size, sizeof(struct sk_buff *),
264 - dev_err(kdev, "cannot allocate rx skb queue\n");
265 + dev_err(kdev, "cannot allocate tx skb queue\n");
267 goto out_free_tx_ring;
269 @@ -2147,11 +2157,11 @@ static int bcm_enetsw_open(struct net_de
270 priv->tx_curr_desc = 0;
271 spin_lock_init(&priv->tx_lock);
273 - /* init & fill rx ring with skbs */
274 - priv->rx_skb = kcalloc(priv->rx_ring_size, sizeof(struct sk_buff *),
275 + /* init & fill rx ring with buffers */
276 + priv->rx_buf = kcalloc(priv->rx_ring_size, sizeof(void *),
278 - if (!priv->rx_skb) {
279 - dev_err(kdev, "cannot allocate rx skb queue\n");
280 + if (!priv->rx_buf) {
281 + dev_err(kdev, "cannot allocate rx buffer queue\n");
283 goto out_free_tx_skb;
285 @@ -2198,8 +2208,8 @@ static int bcm_enetsw_open(struct net_de
286 enet_dma_writel(priv, ENETDMA_BUFALLOC_FORCE_MASK | 0,
287 ENETDMA_BUFALLOC_REG(priv->rx_chan));
289 - if (bcm_enet_refill_rx(dev)) {
290 - dev_err(kdev, "cannot allocate rx skb queue\n");
291 + if (bcm_enet_refill_rx(dev, false)) {
292 + dev_err(kdev, "cannot allocate rx buffer queue\n");
296 @@ -2298,7 +2308,7 @@ static int bcm_enetsw_open(struct net_de
300 - bcm_enet_free_rx_skb_ring(kdev, priv);
301 + bcm_enet_free_rx_buf_ring(kdev, priv);
305 @@ -2348,8 +2358,8 @@ static int bcm_enetsw_stop(struct net_de
306 /* force reclaim of all tx buffers */
307 bcm_enet_tx_reclaim(dev, 1);
309 - /* free the rx skb ring */
310 - bcm_enet_free_rx_skb_ring(kdev, priv);
311 + /* free the rx buffer ring */
312 + bcm_enet_free_rx_buf_ring(kdev, priv);
314 /* free remaining allocated memory */
316 @@ -2648,6 +2658,7 @@ static int bcm_enetsw_probe(struct platf
317 priv->rx_ring_size = BCMENET_DEF_RX_DESC;
318 priv->tx_ring_size = BCMENET_DEF_TX_DESC;
319 priv->dma_maxburst = BCMENETSW_DMA_MAXBURST;
320 + priv->rx_buf_offset = NET_SKB_PAD + NET_IP_ALIGN;
322 pd = dev_get_platdata(&pdev->dev);
324 --- a/drivers/net/ethernet/broadcom/bcm63xx_enet.h
325 +++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
326 @@ -230,11 +230,17 @@ struct bcm_enet_priv {
327 /* next dirty rx descriptor to refill */
330 - /* size of allocated rx skbs */
331 - unsigned int rx_skb_size;
332 + /* size of allocated rx buffers */
333 + unsigned int rx_buf_size;
335 - /* list of skb given to hw for rx */
336 - struct sk_buff **rx_skb;
337 + /* allocated rx buffer offset */
338 + unsigned int rx_buf_offset;
340 + /* size of allocated rx frag */
341 + unsigned int rx_frag_size;
343 + /* list of buffer given to hw for rx */
346 /* used when rx skb allocation failed, so we defer rx queue