target/linux/generic/backport-5.15/606-v5.18-xdp-add-frags-support-to-xdp_return_-buff-frame.patch

   1 commit 7c48cb0176c6d6d3b55029f7ff4ffa05faee6446
   2 Author: Lorenzo Bianconi <lorenzo@kernel.org>
   3 Date:   Fri Jan 21 11:09:50 2022 +0100
   4
   5     xdp: add frags support to xdp_return_{buff/frame}
   6
   7     Take into account if the received xdp_buff/xdp_frame is non-linear
   8     recycling/returning the frame memory to the allocator or into
   9     xdp_frame_bulk.
  10
  11     Acked-by: Toke Hoiland-Jorgensen <toke@redhat.com>
  12     Acked-by: John Fastabend <john.fastabend@gmail.com>
  13     Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
  14     Link: https://lore.kernel.org/r/a961069febc868508ce1bdf5e53a343eb4e57cb2.1642758637.git.lorenzo@kernel.org
  15     Signed-off-by: Alexei Starovoitov <ast@kernel.org>
  16
  17 --- a/include/net/xdp.h
  18 +++ b/include/net/xdp.h
  19 @@ -275,10 +275,24 @@ void __xdp_release_frame(void *data, str
  20  static inline void xdp_release_frame(struct xdp_frame *xdpf)
  21  {
  22         struct xdp_mem_info *mem = &xdpf->mem;
  23 +       struct skb_shared_info *sinfo;
  24 +       int i;
  25
  26         /* Curr only page_pool needs this */
  27 -       if (mem->type == MEM_TYPE_PAGE_POOL)
  28 -               __xdp_release_frame(xdpf->data, mem);
  29 +       if (mem->type != MEM_TYPE_PAGE_POOL)
  30 +               return;
  31 +
  32 +       if (likely(!xdp_frame_has_frags(xdpf)))
  33 +               goto out;
  34 +
  35 +       sinfo = xdp_get_shared_info_from_frame(xdpf);
  36 +       for (i = 0; i < sinfo->nr_frags; i++) {
  37 +               struct page *page = skb_frag_page(&sinfo->frags[i]);
  38 +
  39 +               __xdp_release_frame(page_address(page), mem);
  40 +       }
  41 +out:
  42 +       __xdp_release_frame(xdpf->data, mem);
  43  }
  44
  45  int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
  46 --- a/net/core/xdp.c
  47 +++ b/net/core/xdp.c
  48 @@ -376,12 +376,38 @@ static void __xdp_return(void *data, str
  49
  50  void xdp_return_frame(struct xdp_frame *xdpf)
  51  {
  52 +       struct skb_shared_info *sinfo;
  53 +       int i;
  54 +
  55 +       if (likely(!xdp_frame_has_frags(xdpf)))
  56 +               goto out;
  57 +
  58 +       sinfo = xdp_get_shared_info_from_frame(xdpf);
  59 +       for (i = 0; i < sinfo->nr_frags; i++) {
  60 +               struct page *page = skb_frag_page(&sinfo->frags[i]);
  61 +
  62 +               __xdp_return(page_address(page), &xdpf->mem, false, NULL);
  63 +       }
  64 +out:
  65         __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
  66  }
  67  EXPORT_SYMBOL_GPL(xdp_return_frame);
  68
  69  void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
  70  {
  71 +       struct skb_shared_info *sinfo;
  72 +       int i;
  73 +
  74 +       if (likely(!xdp_frame_has_frags(xdpf)))
  75 +               goto out;
  76 +
  77 +       sinfo = xdp_get_shared_info_from_frame(xdpf);
  78 +       for (i = 0; i < sinfo->nr_frags; i++) {
  79 +               struct page *page = skb_frag_page(&sinfo->frags[i]);
  80 +
  81 +               __xdp_return(page_address(page), &xdpf->mem, true, NULL);
  82 +       }
  83 +out:
  84         __xdp_return(xdpf->data, &xdpf->mem, true, NULL);
  85  }
  86  EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
  87 @@ -417,7 +443,7 @@ void xdp_return_frame_bulk(struct xdp_fr
  88         struct xdp_mem_allocator *xa;
  89
  90         if (mem->type != MEM_TYPE_PAGE_POOL) {
  91 -               __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
  92 +               xdp_return_frame(xdpf);
  93                 return;
  94         }
  95
  96 @@ -436,12 +462,38 @@ void xdp_return_frame_bulk(struct xdp_fr
  97                 bq->xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
  98         }
  99
 100 +       if (unlikely(xdp_frame_has_frags(xdpf))) {
 101 +               struct skb_shared_info *sinfo;
 102 +               int i;
 103 +
 104 +               sinfo = xdp_get_shared_info_from_frame(xdpf);
 105 +               for (i = 0; i < sinfo->nr_frags; i++) {
 106 +                       skb_frag_t *frag = &sinfo->frags[i];
 107 +
 108 +                       bq->q[bq->count++] = skb_frag_address(frag);
 109 +                       if (bq->count == XDP_BULK_QUEUE_SIZE)
 110 +                               xdp_flush_frame_bulk(bq);
 111 +               }
 112 +       }
 113         bq->q[bq->count++] = xdpf->data;
 114  }
 115  EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);
 116
 117  void xdp_return_buff(struct xdp_buff *xdp)
 118  {
 119 +       struct skb_shared_info *sinfo;
 120 +       int i;
 121 +
 122 +       if (likely(!xdp_buff_has_frags(xdp)))
 123 +               goto out;
 124 +
 125 +       sinfo = xdp_get_shared_info_from_buff(xdp);
 126 +       for (i = 0; i < sinfo->nr_frags; i++) {
 127 +               struct page *page = skb_frag_page(&sinfo->frags[i]);
 128 +
 129 +               __xdp_return(page_address(page), &xdp->rxq->mem, true, xdp);
 130 +       }
 131 +out:
 132         __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp);
 133  }
 134