target/linux/mvebu/patches-5.4/012-net-mvneta-rely-on-build_skb-in-mvneta_rx_swbm-poll-.patch

   1 From 00cf2a1d7d58631ba137b9acabe1de1d542625a8 Mon Sep 17 00:00:00 2001
   2 From: Lorenzo Bianconi <lorenzo@kernel.org>
   3 Date: Sat, 19 Oct 2019 10:13:23 +0200
   4 Subject: [PATCH 3/7] net: mvneta: rely on build_skb in mvneta_rx_swbm poll
   5  routine
   6
   7 Refactor mvneta_rx_swbm code introducing mvneta_swbm_rx_frame and
   8 mvneta_swbm_add_rx_fragment routines. Rely on build_skb in oreder to
   9 allocate skb since the previous patch introduced buffer recycling using
  10 the page_pool API.
  11 This patch fixes even an issue in the original driver where dma buffers
  12 are accessed before dma sync.
  13 mvneta driver can run on not cache coherent devices so it is
  14 necessary to sync DMA buffers before sending them to the device
  15 in order to avoid memory corruptions. Running perf analysis we can
  16 see a performance cost associated with this DMA-sync (anyway it is
  17 already there in the original driver code). In follow up patches we
  18 will add more logic to reduce DMA-sync as much as possible.
  19
  20 Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
  21 Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
  22 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
  23 Signed-off-by: David S. Miller <davem@davemloft.net>
  24 ---
  25  drivers/net/ethernet/marvell/mvneta.c | 185 +++++++++++++-------------
  26  1 file changed, 95 insertions(+), 90 deletions(-)
  27
  28 --- a/drivers/net/ethernet/marvell/mvneta.c
  29 +++ b/drivers/net/ethernet/marvell/mvneta.c
  30 @@ -325,6 +325,11 @@
  31               ETH_HLEN + ETH_FCS_LEN,                        \
  32               cache_line_size())
  33
  34 +#define MVNETA_SKB_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info) + \
  35 +                        NET_SKB_PAD))
  36 +#define MVNETA_SKB_SIZE(len)   (SKB_DATA_ALIGN(len) + MVNETA_SKB_PAD)
  37 +#define MVNETA_MAX_RX_BUF_SIZE (PAGE_SIZE - MVNETA_SKB_PAD)
  38 +
  39  #define IS_TSO_HEADER(txq, addr) \
  40         ((addr >= txq->tso_hdrs_phys) && \
  41          (addr < txq->tso_hdrs_phys + txq->size * TSO_HEADER_SIZE))
  42 @@ -650,7 +655,6 @@ static int txq_number = 8;
  43  static int rxq_def;
  44
  45  static int rx_copybreak __read_mostly = 256;
  46 -static int rx_header_size __read_mostly = 128;
  47
  48  /* HW BM need that each port be identify by a unique ID */
  49  static int global_port_id;
  50 @@ -1842,7 +1846,7 @@ static int mvneta_rx_refill(struct mvnet
  51         phys_addr = page_pool_get_dma_addr(page) + pp->rx_offset_correction;
  52         dma_dir = page_pool_get_dma_dir(rxq->page_pool);
  53         dma_sync_single_for_device(pp->dev->dev.parent, phys_addr,
  54 -                                  PAGE_SIZE, dma_dir);
  55 +                                  MVNETA_MAX_RX_BUF_SIZE, dma_dir);
  56         mvneta_rx_desc_fill(rx_desc, phys_addr, page, rxq);
  57
  58         return 0;
  59 @@ -1960,30 +1964,102 @@ int mvneta_rx_refill_queue(struct mvneta
  60         return i;
  61  }
  62
  63 +static int
  64 +mvneta_swbm_rx_frame(struct mvneta_port *pp,
  65 +                    struct mvneta_rx_desc *rx_desc,
  66 +                    struct mvneta_rx_queue *rxq,
  67 +                    struct page *page)
  68 +{
  69 +       unsigned char *data = page_address(page);
  70 +       int data_len = -MVNETA_MH_SIZE, len;
  71 +       struct net_device *dev = pp->dev;
  72 +       enum dma_data_direction dma_dir;
  73 +
  74 +       if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) {
  75 +               len = MVNETA_MAX_RX_BUF_SIZE;
  76 +               data_len += len;
  77 +       } else {
  78 +               len = rx_desc->data_size;
  79 +               data_len += len - ETH_FCS_LEN;
  80 +       }
  81 +
  82 +       dma_dir = page_pool_get_dma_dir(rxq->page_pool);
  83 +       dma_sync_single_for_cpu(dev->dev.parent,
  84 +                               rx_desc->buf_phys_addr,
  85 +                               len, dma_dir);
  86 +
  87 +       rxq->skb = build_skb(data, PAGE_SIZE);
  88 +       if (unlikely(!rxq->skb)) {
  89 +               netdev_err(dev,
  90 +                          "Can't allocate skb on queue %d\n",
  91 +                          rxq->id);
  92 +               dev->stats.rx_dropped++;
  93 +               rxq->skb_alloc_err++;
  94 +               return -ENOMEM;
  95 +       }
  96 +       page_pool_release_page(rxq->page_pool, page);
  97 +
  98 +       skb_reserve(rxq->skb, MVNETA_MH_SIZE + NET_SKB_PAD);
  99 +       skb_put(rxq->skb, data_len);
 100 +       mvneta_rx_csum(pp, rx_desc->status, rxq->skb);
 101 +
 102 +       rxq->left_size = rx_desc->data_size - len;
 103 +       rx_desc->buf_phys_addr = 0;
 104 +
 105 +       return 0;
 106 +}
 107 +
 108 +static void
 109 +mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,
 110 +                           struct mvneta_rx_desc *rx_desc,
 111 +                           struct mvneta_rx_queue *rxq,
 112 +                           struct page *page)
 113 +{
 114 +       struct net_device *dev = pp->dev;
 115 +       enum dma_data_direction dma_dir;
 116 +       int data_len, len;
 117 +
 118 +       if (rxq->left_size > MVNETA_MAX_RX_BUF_SIZE) {
 119 +               len = MVNETA_MAX_RX_BUF_SIZE;
 120 +               data_len = len;
 121 +       } else {
 122 +               len = rxq->left_size;
 123 +               data_len = len - ETH_FCS_LEN;
 124 +       }
 125 +       dma_dir = page_pool_get_dma_dir(rxq->page_pool);
 126 +       dma_sync_single_for_cpu(dev->dev.parent,
 127 +                               rx_desc->buf_phys_addr,
 128 +                               len, dma_dir);
 129 +       if (data_len > 0) {
 130 +               /* refill descriptor with new buffer later */
 131 +               skb_add_rx_frag(rxq->skb,
 132 +                               skb_shinfo(rxq->skb)->nr_frags,
 133 +                               page, NET_SKB_PAD, data_len,
 134 +                               PAGE_SIZE);
 135 +       }
 136 +       page_pool_release_page(rxq->page_pool, page);
 137 +       rx_desc->buf_phys_addr = 0;
 138 +       rxq->left_size -= len;
 139 +}
 140 +
 141  /* Main rx processing when using software buffer management */
 142  static int mvneta_rx_swbm(struct napi_struct *napi,
 143                           struct mvneta_port *pp, int budget,
 144                           struct mvneta_rx_queue *rxq)
 145  {
 146 +       int rcvd_pkts = 0, rcvd_bytes = 0, rx_proc = 0;
 147         struct net_device *dev = pp->dev;
 148 -       int rx_todo, rx_proc;
 149 -       int refill = 0;
 150 -       u32 rcvd_pkts = 0;
 151 -       u32 rcvd_bytes = 0;
 152 +       int rx_todo, refill;
 153
 154         /* Get number of received packets */
 155         rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
 156 -       rx_proc = 0;
 157
 158         /* Fairness NAPI loop */
 159 -       while ((rcvd_pkts < budget) && (rx_proc < rx_todo)) {
 160 +       while (rx_proc < budget && rx_proc < rx_todo) {
 161                 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
 162 +               u32 rx_status, index;
 163                 unsigned char *data;
 164                 struct page *page;
 165 -               dma_addr_t phys_addr;
 166 -               u32 rx_status, index;
 167 -               int rx_bytes, skb_size, copy_size;
 168 -               int frag_num, frag_size, frag_offset;
 169
 170                 index = rx_desc - rxq->descs;
 171                 page = (struct page *)rxq->buf_virt_addr[index];
 172 @@ -1991,100 +2067,30 @@ static int mvneta_rx_swbm(struct napi_st
 173                 /* Prefetch header */
 174                 prefetch(data);
 175
 176 -               phys_addr = rx_desc->buf_phys_addr;
 177                 rx_status = rx_desc->status;
 178                 rx_proc++;
 179                 rxq->refill_num++;
 180
 181                 if (rx_status & MVNETA_RXD_FIRST_DESC) {
 182 +                       int err;
 183 +
 184                         /* Check errors only for FIRST descriptor */
 185                         if (rx_status & MVNETA_RXD_ERR_SUMMARY) {
 186                                 mvneta_rx_error(pp, rx_desc);
 187                                 /* leave the descriptor untouched */
 188                                 continue;
 189                         }
 190 -                       rx_bytes = rx_desc->data_size -
 191 -                                  (ETH_FCS_LEN + MVNETA_MH_SIZE);
 192 -
 193 -                       /* Allocate small skb for each new packet */
 194 -                       skb_size = max(rx_copybreak, rx_header_size);
 195 -                       rxq->skb = netdev_alloc_skb_ip_align(dev, skb_size);
 196 -                       if (unlikely(!rxq->skb)) {
 197 -                               struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
 198 -
 199 -                               netdev_err(dev,
 200 -                                          "Can't allocate skb on queue %d\n",
 201 -                                          rxq->id);
 202 -
 203 -                               rxq->skb_alloc_err++;
 204
 205 -                               u64_stats_update_begin(&stats->syncp);
 206 -                               stats->rx_dropped++;
 207 -                               u64_stats_update_end(&stats->syncp);
 208 +                       err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, page);
 209 +                       if (err)
 210                                 continue;
 211 -                       }
 212 -                       copy_size = min(skb_size, rx_bytes);
 213 -
 214 -                       /* Copy data from buffer to SKB, skip Marvell header */
 215 -                       memcpy(rxq->skb->data, data + MVNETA_MH_SIZE,
 216 -                              copy_size);
 217 -                       skb_put(rxq->skb, copy_size);
 218 -                       rxq->left_size = rx_bytes - copy_size;
 219 -
 220 -                       mvneta_rx_csum(pp, rx_status, rxq->skb);
 221 -                       if (rxq->left_size == 0) {
 222 -                               int size = copy_size + MVNETA_MH_SIZE;
 223 -
 224 -                               dma_sync_single_range_for_cpu(dev->dev.parent,
 225 -                                                             phys_addr, 0,
 226 -                                                             size,
 227 -                                                             DMA_FROM_DEVICE);
 228 -
 229 -                               /* leave the descriptor and buffer untouched */
 230 -                       } else {
 231 -                               /* refill descriptor with new buffer later */
 232 -                               rx_desc->buf_phys_addr = 0;
 233 -
 234 -                               frag_num = 0;
 235 -                               frag_offset = copy_size + MVNETA_MH_SIZE;
 236 -                               frag_size = min(rxq->left_size,
 237 -                                               (int)(PAGE_SIZE - frag_offset));
 238 -                               skb_add_rx_frag(rxq->skb, frag_num, page,
 239 -                                               frag_offset, frag_size,
 240 -                                               PAGE_SIZE);
 241 -                               page_pool_release_page(rxq->page_pool, page);
 242 -                               rxq->left_size -= frag_size;
 243 -                       }
 244                 } else {
 245 -                       /* Middle or Last descriptor */
 246                         if (unlikely(!rxq->skb)) {
 247                                 pr_debug("no skb for rx_status 0x%x\n",
 248                                          rx_status);
 249                                 continue;
 250                         }
 251 -                       if (!rxq->left_size) {
 252 -                               /* last descriptor has only FCS */
 253 -                               /* and can be discarded */
 254 -                               dma_sync_single_range_for_cpu(dev->dev.parent,
 255 -                                                             phys_addr, 0,
 256 -                                                             ETH_FCS_LEN,
 257 -                                                             DMA_FROM_DEVICE);
 258 -                               /* leave the descriptor and buffer untouched */
 259 -                       } else {
 260 -                               /* refill descriptor with new buffer later */
 261 -                               rx_desc->buf_phys_addr = 0;
 262 -
 263 -                               frag_num = skb_shinfo(rxq->skb)->nr_frags;
 264 -                               frag_offset = 0;
 265 -                               frag_size = min(rxq->left_size,
 266 -                                               (int)(PAGE_SIZE - frag_offset));
 267 -                               skb_add_rx_frag(rxq->skb, frag_num, page,
 268 -                                               frag_offset, frag_size,
 269 -                                               PAGE_SIZE);
 270 -
 271 -                               page_pool_release_page(rxq->page_pool, page);
 272 -                               rxq->left_size -= frag_size;
 273 -                       }
 274 +                       mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, page);
 275                 } /* Middle or Last descriptor */
 276
 277                 if (!(rx_status & MVNETA_RXD_LAST_DESC))
 278 @@ -2109,7 +2115,6 @@ static int mvneta_rx_swbm(struct napi_st
 279
 280                 /* clean uncomplete skb pointer in queue */
 281                 rxq->skb = NULL;
 282 -               rxq->left_size = 0;
 283         }
 284
 285         if (rcvd_pkts)
 286 @@ -2970,7 +2975,7 @@ static void mvneta_rxq_hw_init(struct mv
 287                 /* Set Offset */
 288                 mvneta_rxq_offset_set(pp, rxq, 0);
 289                 mvneta_rxq_buf_size_set(pp, rxq, PAGE_SIZE < SZ_64K ?
 290 -                                       PAGE_SIZE :
 291 +                                       MVNETA_MAX_RX_BUF_SIZE :
 292                                         MVNETA_RX_BUF_SIZE(pp->pkt_size));
 293                 mvneta_rxq_bm_disable(pp, rxq);
 294                 mvneta_rxq_fill(pp, rxq, rxq->size);
 295 @@ -4708,7 +4713,7 @@ static int mvneta_probe(struct platform_
 296         SET_NETDEV_DEV(dev, &pdev->dev);
 297
 298         pp->id = global_port_id++;
 299 -       pp->rx_offset_correction = 0; /* not relevant for SW BM */
 300 +       pp->rx_offset_correction = NET_SKB_PAD;
 301
 302         /* Obtain access to BM resources if enabled and already initialized */
 303         bm_node = of_parse_phandle(dn, "buffer-manager", 0);