1 commit 7c48cb0176c6d6d3b55029f7ff4ffa05faee6446
2 Author: Lorenzo Bianconi <lorenzo@kernel.org>
3 Date: Fri Jan 21 11:09:50 2022 +0100
5 xdp: add frags support to xdp_return_{buff/frame}
7 Take into account if the received xdp_buff/xdp_frame is non-linear
8 recycling/returning the frame memory to the allocator or into
11 Acked-by: Toke Hoiland-Jorgensen <toke@redhat.com>
12 Acked-by: John Fastabend <john.fastabend@gmail.com>
13 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
14 Link: https://lore.kernel.org/r/a961069febc868508ce1bdf5e53a343eb4e57cb2.1642758637.git.lorenzo@kernel.org
15 Signed-off-by: Alexei Starovoitov <ast@kernel.org>
17 --- a/include/net/xdp.h
18 +++ b/include/net/xdp.h
19 @@ -275,10 +275,24 @@ void __xdp_release_frame(void *data, str
20 static inline void xdp_release_frame(struct xdp_frame *xdpf)
22 struct xdp_mem_info *mem = &xdpf->mem;
23 + struct skb_shared_info *sinfo;
26 /* Curr only page_pool needs this */
27 - if (mem->type == MEM_TYPE_PAGE_POOL)
28 - __xdp_release_frame(xdpf->data, mem);
29 + if (mem->type != MEM_TYPE_PAGE_POOL)
32 + if (likely(!xdp_frame_has_frags(xdpf)))
35 + sinfo = xdp_get_shared_info_from_frame(xdpf);
36 + for (i = 0; i < sinfo->nr_frags; i++) {
37 + struct page *page = skb_frag_page(&sinfo->frags[i]);
39 + __xdp_release_frame(page_address(page), mem);
42 + __xdp_release_frame(xdpf->data, mem);
45 int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
48 @@ -376,12 +376,38 @@ static void __xdp_return(void *data, str
50 void xdp_return_frame(struct xdp_frame *xdpf)
52 + struct skb_shared_info *sinfo;
55 + if (likely(!xdp_frame_has_frags(xdpf)))
58 + sinfo = xdp_get_shared_info_from_frame(xdpf);
59 + for (i = 0; i < sinfo->nr_frags; i++) {
60 + struct page *page = skb_frag_page(&sinfo->frags[i]);
62 + __xdp_return(page_address(page), &xdpf->mem, false, NULL);
65 __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
67 EXPORT_SYMBOL_GPL(xdp_return_frame);
69 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
71 + struct skb_shared_info *sinfo;
74 + if (likely(!xdp_frame_has_frags(xdpf)))
77 + sinfo = xdp_get_shared_info_from_frame(xdpf);
78 + for (i = 0; i < sinfo->nr_frags; i++) {
79 + struct page *page = skb_frag_page(&sinfo->frags[i]);
81 + __xdp_return(page_address(page), &xdpf->mem, true, NULL);
84 __xdp_return(xdpf->data, &xdpf->mem, true, NULL);
86 EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
87 @@ -417,7 +443,7 @@ void xdp_return_frame_bulk(struct xdp_fr
88 struct xdp_mem_allocator *xa;
90 if (mem->type != MEM_TYPE_PAGE_POOL) {
91 - __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
92 + xdp_return_frame(xdpf);
96 @@ -436,12 +462,38 @@ void xdp_return_frame_bulk(struct xdp_fr
97 bq->xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
100 + if (unlikely(xdp_frame_has_frags(xdpf))) {
101 + struct skb_shared_info *sinfo;
104 + sinfo = xdp_get_shared_info_from_frame(xdpf);
105 + for (i = 0; i < sinfo->nr_frags; i++) {
106 + skb_frag_t *frag = &sinfo->frags[i];
108 + bq->q[bq->count++] = skb_frag_address(frag);
109 + if (bq->count == XDP_BULK_QUEUE_SIZE)
110 + xdp_flush_frame_bulk(bq);
113 bq->q[bq->count++] = xdpf->data;
115 EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);
117 void xdp_return_buff(struct xdp_buff *xdp)
119 + struct skb_shared_info *sinfo;
122 + if (likely(!xdp_buff_has_frags(xdp)))
125 + sinfo = xdp_get_shared_info_from_buff(xdp);
126 + for (i = 0; i < sinfo->nr_frags; i++) {
127 + struct page *page = skb_frag_page(&sinfo->frags[i]);
129 + __xdp_return(page_address(page), &xdp->rxq->mem, true, xdp);
132 __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp);