target/linux/generic/backport-5.15/606-v5.18-xdp-add-frags-support-to-xdp_return_-buff-frame.patch

   1 From 7c48cb0176c6d6d3b55029f7ff4ffa05faee6446 Mon Sep 17 00:00:00 2001
   2 From: Lorenzo Bianconi <lorenzo@kernel.org>
   3 Date: Fri, 21 Jan 2022 11:09:50 +0100
   4 Subject: [PATCH] xdp: add frags support to xdp_return_{buff/frame}
   5
   6 Take into account if the received xdp_buff/xdp_frame is non-linear
   7 recycling/returning the frame memory to the allocator or into
   8 xdp_frame_bulk.
   9
  10 Acked-by: Toke Hoiland-Jorgensen <toke@redhat.com>
  11 Acked-by: John Fastabend <john.fastabend@gmail.com>
  12 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
  13 Link: https://lore.kernel.org/r/a961069febc868508ce1bdf5e53a343eb4e57cb2.1642758637.git.lorenzo@kernel.org
  14 Signed-off-by: Alexei Starovoitov <ast@kernel.org>
  15 ---
  16  include/net/xdp.h | 18 ++++++++++++++--
  17  net/core/xdp.c    | 54 ++++++++++++++++++++++++++++++++++++++++++++++-
  18  2 files changed, 69 insertions(+), 3 deletions(-)
  19
  20 diff --git a/include/net/xdp.h b/include/net/xdp.h
  21 index 1f8641ec658e..8463dea8b4db 100644
  22 --- a/include/net/xdp.h
  23 +++ b/include/net/xdp.h
  24 @@ -275,10 +275,24 @@ void __xdp_release_frame(void *data, str
  25  static inline void xdp_release_frame(struct xdp_frame *xdpf)
  26  {
  27         struct xdp_mem_info *mem = &xdpf->mem;
  28 +       struct skb_shared_info *sinfo;
  29 +       int i;
  30
  31         /* Curr only page_pool needs this */
  32 -       if (mem->type == MEM_TYPE_PAGE_POOL)
  33 -               __xdp_release_frame(xdpf->data, mem);
  34 +       if (mem->type != MEM_TYPE_PAGE_POOL)
  35 +               return;
  36 +
  37 +       if (likely(!xdp_frame_has_frags(xdpf)))
  38 +               goto out;
  39 +
  40 +       sinfo = xdp_get_shared_info_from_frame(xdpf);
  41 +       for (i = 0; i < sinfo->nr_frags; i++) {
  42 +               struct page *page = skb_frag_page(&sinfo->frags[i]);
  43 +
  44 +               __xdp_release_frame(page_address(page), mem);
  45 +       }
  46 +out:
  47 +       __xdp_release_frame(xdpf->data, mem);
  48  }
  49
  50  int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
  51 --- a/net/core/xdp.c
  52 +++ b/net/core/xdp.c
  53 @@ -376,12 +376,38 @@ static void __xdp_return(void *data, str
  54
  55  void xdp_return_frame(struct xdp_frame *xdpf)
  56  {
  57 +       struct skb_shared_info *sinfo;
  58 +       int i;
  59 +
  60 +       if (likely(!xdp_frame_has_frags(xdpf)))
  61 +               goto out;
  62 +
  63 +       sinfo = xdp_get_shared_info_from_frame(xdpf);
  64 +       for (i = 0; i < sinfo->nr_frags; i++) {
  65 +               struct page *page = skb_frag_page(&sinfo->frags[i]);
  66 +
  67 +               __xdp_return(page_address(page), &xdpf->mem, false, NULL);
  68 +       }
  69 +out:
  70         __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
  71  }
  72  EXPORT_SYMBOL_GPL(xdp_return_frame);
  73
  74  void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
  75  {
  76 +       struct skb_shared_info *sinfo;
  77 +       int i;
  78 +
  79 +       if (likely(!xdp_frame_has_frags(xdpf)))
  80 +               goto out;
  81 +
  82 +       sinfo = xdp_get_shared_info_from_frame(xdpf);
  83 +       for (i = 0; i < sinfo->nr_frags; i++) {
  84 +               struct page *page = skb_frag_page(&sinfo->frags[i]);
  85 +
  86 +               __xdp_return(page_address(page), &xdpf->mem, true, NULL);
  87 +       }
  88 +out:
  89         __xdp_return(xdpf->data, &xdpf->mem, true, NULL);
  90  }
  91  EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
  92 @@ -417,7 +443,7 @@ void xdp_return_frame_bulk(struct xdp_fr
  93         struct xdp_mem_allocator *xa;
  94
  95         if (mem->type != MEM_TYPE_PAGE_POOL) {
  96 -               __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
  97 +               xdp_return_frame(xdpf);
  98                 return;
  99         }
 100
 101 @@ -436,12 +462,38 @@ void xdp_return_frame_bulk(struct xdp_fr
 102                 bq->xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
 103         }
 104
 105 +       if (unlikely(xdp_frame_has_frags(xdpf))) {
 106 +               struct skb_shared_info *sinfo;
 107 +               int i;
 108 +
 109 +               sinfo = xdp_get_shared_info_from_frame(xdpf);
 110 +               for (i = 0; i < sinfo->nr_frags; i++) {
 111 +                       skb_frag_t *frag = &sinfo->frags[i];
 112 +
 113 +                       bq->q[bq->count++] = skb_frag_address(frag);
 114 +                       if (bq->count == XDP_BULK_QUEUE_SIZE)
 115 +                               xdp_flush_frame_bulk(bq);
 116 +               }
 117 +       }
 118         bq->q[bq->count++] = xdpf->data;
 119  }
 120  EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);
 121
 122  void xdp_return_buff(struct xdp_buff *xdp)
 123  {
 124 +       struct skb_shared_info *sinfo;
 125 +       int i;
 126 +
 127 +       if (likely(!xdp_buff_has_frags(xdp)))
 128 +               goto out;
 129 +
 130 +       sinfo = xdp_get_shared_info_from_buff(xdp);
 131 +       for (i = 0; i < sinfo->nr_frags; i++) {
 132 +               struct page *page = skb_frag_page(&sinfo->frags[i]);
 133 +
 134 +               __xdp_return(page_address(page), &xdp->rxq->mem, true, xdp);
 135 +       }
 136 +out:
 137         __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp);
 138  }
 139