1 --- a/include/net/page_pool.h
2 +++ b/include/net/page_pool.h
4 * Please note DMA-sync-for-CPU is still
5 * device driver responsibility
7 -#define PP_FLAG_ALL (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV)
8 +#define PP_FLAG_PAGE_FRAG BIT(2) /* for page frag feature */
9 +#define PP_FLAG_ALL (PP_FLAG_DMA_MAP |\
10 + PP_FLAG_DMA_SYNC_DEV |\
14 * Fast allocation side cache array/stack
16 #define PP_ALLOC_CACHE_REFILL 64
17 struct pp_alloc_cache {
19 - void *cache[PP_ALLOC_CACHE_SIZE];
20 + struct page *cache[PP_ALLOC_CACHE_SIZE];
23 struct page_pool_params {
24 @@ -79,6 +82,22 @@ struct page_pool_params {
25 unsigned int offset; /* DMA addr offset */
29 +static inline int page_pool_ethtool_stats_get_count(void)
34 +static inline u8 *page_pool_ethtool_stats_get_strings(u8 *data)
39 +static inline u64 *page_pool_ethtool_stats_get(u64 *data, void *stats)
45 struct page_pool_params p;
47 @@ -88,6 +107,9 @@ struct page_pool {
48 unsigned long defer_warn;
50 u32 pages_state_hold_cnt;
51 + unsigned int frag_offset;
52 + struct page *frag_page;
56 * Data structure for allocation side
57 @@ -137,6 +159,18 @@ static inline struct page *page_pool_dev
58 return page_pool_alloc_pages(pool, gfp);
61 +struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
62 + unsigned int size, gfp_t gfp);
64 +static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool,
65 + unsigned int *offset,
68 + gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
70 + return page_pool_alloc_frag(pool, offset, size, gfp);
73 /* get the stored dma direction. A driver might decide to treat this locally and
74 * avoid the extra cache line from page_pool to determine the direction
76 @@ -146,6 +180,8 @@ inline enum dma_data_direction page_pool
77 return pool->p.dma_dir;
80 +bool page_pool_return_skb_page(struct page *page);
82 struct page_pool *page_pool_create(const struct page_pool_params *params);
84 #ifdef CONFIG_PAGE_POOL
85 @@ -165,6 +201,7 @@ static inline void page_pool_release_pag
92 void page_pool_put_page(struct page_pool *pool, struct page *page,
93 @@ -189,19 +226,48 @@ static inline void page_pool_recycle_dir
94 page_pool_put_full_page(pool, page, true);
97 +#define PAGE_POOL_DMA_USE_PP_FRAG_COUNT \
98 + (sizeof(dma_addr_t) > sizeof(unsigned long))
100 static inline dma_addr_t page_pool_get_dma_addr(struct page *page)
102 - dma_addr_t ret = page->dma_addr[0];
103 - if (sizeof(dma_addr_t) > sizeof(unsigned long))
104 - ret |= (dma_addr_t)page->dma_addr[1] << 16 << 16;
105 + dma_addr_t ret = page->dma_addr;
107 + if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT)
108 + ret |= (dma_addr_t)page->dma_addr_upper << 16 << 16;
113 static inline void page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
115 - page->dma_addr[0] = addr;
116 - if (sizeof(dma_addr_t) > sizeof(unsigned long))
117 - page->dma_addr[1] = upper_32_bits(addr);
118 + page->dma_addr = addr;
119 + if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT)
120 + page->dma_addr_upper = upper_32_bits(addr);
123 +static inline void page_pool_set_frag_count(struct page *page, long nr)
125 + atomic_long_set(&page->pp_frag_count, nr);
128 +static inline long page_pool_atomic_sub_frag_count_return(struct page *page,
133 + /* As suggested by Alexander, atomic_long_read() may cover up the
134 + * reference count errors, so avoid calling atomic_long_read() in
135 + * the cases of freeing or draining the page_frags, where we would
136 + * not expect it to match or that are slowpath anyway.
138 + if (__builtin_constant_p(nr) &&
139 + atomic_long_read(&page->pp_frag_count) == nr)
142 + ret = atomic_long_sub_return(nr, &page->pp_frag_count);
147 static inline bool is_page_pool_compiled_in(void)
148 @@ -225,4 +291,23 @@ static inline void page_pool_nid_changed
149 if (unlikely(pool->p.nid != new_nid))
150 page_pool_update_nid(pool, new_nid);
153 +static inline void page_pool_ring_lock(struct page_pool *pool)
154 + __acquires(&pool->ring.producer_lock)
156 + if (in_serving_softirq())
157 + spin_lock(&pool->ring.producer_lock);
159 + spin_lock_bh(&pool->ring.producer_lock);
162 +static inline void page_pool_ring_unlock(struct page_pool *pool)
163 + __releases(&pool->ring.producer_lock)
165 + if (in_serving_softirq())
166 + spin_unlock(&pool->ring.producer_lock);
168 + spin_unlock_bh(&pool->ring.producer_lock);
171 #endif /* _NET_PAGE_POOL_H */
172 --- a/net/core/page_pool.c
173 +++ b/net/core/page_pool.c
175 #include <linux/device.h>
177 #include <net/page_pool.h>
178 +#include <net/xdp.h>
180 #include <linux/dma-direction.h>
181 #include <linux/dma-mapping.h>
182 #include <linux/page-flags.h>
183 #include <linux/mm.h> /* for __put_page() */
184 +#include <linux/poison.h>
185 +#include <linux/ethtool.h>
187 #include <trace/events/page_pool.h>
189 #define DEFER_TIME (msecs_to_jiffies(1000))
190 #define DEFER_WARN_INTERVAL (60 * HZ)
192 +#define BIAS_MAX LONG_MAX
194 static int page_pool_init(struct page_pool *pool,
195 const struct page_pool_params *params)
197 @@ -64,6 +70,10 @@ static int page_pool_init(struct page_po
201 + if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT &&
202 + pool->p.flags & PP_FLAG_PAGE_FRAG)
205 if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0)
208 @@ -180,40 +190,10 @@ static void page_pool_dma_sync_for_devic
214 -static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
216 +static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
222 - /* We could always set __GFP_COMP, and avoid this branch, as
223 - * prep_new_page() can handle order-0 with __GFP_COMP.
228 - /* FUTURE development:
230 - * Current slow-path essentially falls back to single page
231 - * allocations, which doesn't improve performance. This code
232 - * need bulk allocation support from the page allocator code.
235 - /* Cache was empty, do real allocation */
237 - page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
239 - page = alloc_pages(gfp, pool->p.order);
244 - if (!(pool->p.flags & PP_FLAG_DMA_MAP))
247 /* Setup DMA mapping: use 'struct page' area for storing DMA-addr
248 * since dma_addr_t can be either 32 or 64 bits and does not always fit
249 * into page private data (i.e 32bit cpu with 64bit DMA caps)
250 @@ -222,22 +202,53 @@ static struct page *__page_pool_alloc_pa
251 dma = dma_map_page_attrs(pool->p.dev, page, 0,
252 (PAGE_SIZE << pool->p.order),
253 pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
254 - if (dma_mapping_error(pool->p.dev, dma)) {
258 + if (dma_mapping_error(pool->p.dev, dma))
261 page_pool_set_dma_addr(page, dma);
263 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
264 page_pool_dma_sync_for_device(pool, page, pool->p.max_len);
270 +static void page_pool_set_pp_info(struct page_pool *pool,
274 + page->pp_magic |= PP_SIGNATURE;
277 +static void page_pool_clear_pp_info(struct page *page)
279 + page->pp_magic = 0;
285 +static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
291 + page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
292 + if (unlikely(!page))
295 + if ((pool->p.flags & PP_FLAG_DMA_MAP) &&
296 + unlikely(!page_pool_dma_map(pool, page))) {
301 + page_pool_set_pp_info(pool, page);
303 /* Track how many pages are held 'in-flight' */
304 pool->pages_state_hold_cnt++;
306 trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);
308 - /* When page just alloc'ed is should/must have refcnt 1. */
312 @@ -302,10 +313,12 @@ void page_pool_release_page(struct page_
313 DMA_ATTR_SKIP_CPU_SYNC);
314 page_pool_set_dma_addr(page, 0);
316 + page_pool_clear_pp_info(page);
318 /* This may be the last page returned, releasing the pool, so
319 * it is not safe to reference pool afterwards.
321 - count = atomic_inc_return(&pool->pages_state_release_cnt);
322 + count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt);
323 trace_page_pool_state_release(pool, page, count);
325 EXPORT_SYMBOL(page_pool_release_page);
326 @@ -331,7 +344,10 @@ static bool page_pool_recycle_in_ring(st
328 ret = ptr_ring_produce_bh(&pool->ring, page);
330 - return (ret == 0) ? true : false;
337 /* Only allow direct recycling in special circumstances, into the
338 @@ -350,46 +366,43 @@ static bool page_pool_recycle_in_cache(s
342 -/* page is NOT reusable when:
343 - * 1) allocated when system is under some pressure. (page_is_pfmemalloc)
345 -static bool pool_page_reusable(struct page_pool *pool, struct page *page)
347 - return !page_is_pfmemalloc(page);
350 /* If the page refcnt == 1, this will try to recycle the page.
351 * if PP_FLAG_DMA_SYNC_DEV is set, we'll try to sync the DMA area for
352 * the configured size min(dma_sync_size, pool->max_len).
353 * If the page refcnt != 1, then the page will be returned to memory
356 -void page_pool_put_page(struct page_pool *pool, struct page *page,
357 - unsigned int dma_sync_size, bool allow_direct)
359 +static __always_inline struct page *
360 +__page_pool_put_page(struct page_pool *pool, struct page *page,
361 + unsigned int dma_sync_size, bool allow_direct)
363 + /* It is not the last user for the page frag case */
364 + if (pool->p.flags & PP_FLAG_PAGE_FRAG &&
365 + page_pool_atomic_sub_frag_count_return(page, 1))
368 /* This allocator is optimized for the XDP mode that uses
369 * one-frame-per-page, but have fallbacks that act like the
370 * regular page allocator APIs.
372 * refcnt == 1 means page_pool owns page, and can recycle it.
374 + * page is NOT reusable when allocated when system is under
375 + * some pressure. (page_is_pfmemalloc)
377 - if (likely(page_ref_count(page) == 1 &&
378 - pool_page_reusable(pool, page))) {
379 + if (likely(page_ref_count(page) == 1 && !page_is_pfmemalloc(page))) {
380 /* Read barrier done in page_ref_count / READ_ONCE */
382 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
383 page_pool_dma_sync_for_device(pool, page,
386 - if (allow_direct && in_serving_softirq())
387 - if (page_pool_recycle_in_cache(page, pool))
389 + if (allow_direct && in_serving_softirq() &&
390 + page_pool_recycle_in_cache(page, pool))
393 - if (!page_pool_recycle_in_ring(pool, page)) {
394 - /* Cache full, fallback to free pages */
395 - page_pool_return_page(pool, page);
398 + /* Page found as candidate for recycling */
401 /* Fallback/non-XDP mode: API user have elevated refcnt.
403 @@ -407,9 +420,98 @@ void page_pool_put_page(struct page_pool
404 /* Do not replace this with page_pool_return_page() */
405 page_pool_release_page(pool, page);
411 +void page_pool_put_page(struct page_pool *pool, struct page *page,
412 + unsigned int dma_sync_size, bool allow_direct)
414 + page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct);
415 + if (page && !page_pool_recycle_in_ring(pool, page))
416 + /* Cache full, fallback to free pages */
417 + page_pool_return_page(pool, page);
419 EXPORT_SYMBOL(page_pool_put_page);
421 +static struct page *page_pool_drain_frag(struct page_pool *pool,
424 + long drain_count = BIAS_MAX - pool->frag_users;
426 + /* Some user is still using the page frag */
427 + if (likely(page_pool_atomic_sub_frag_count_return(page,
431 + if (page_ref_count(page) == 1 && !page_is_pfmemalloc(page)) {
432 + if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
433 + page_pool_dma_sync_for_device(pool, page, -1);
438 + page_pool_return_page(pool, page);
442 +static void page_pool_free_frag(struct page_pool *pool)
444 + long drain_count = BIAS_MAX - pool->frag_users;
445 + struct page *page = pool->frag_page;
447 + pool->frag_page = NULL;
450 + page_pool_atomic_sub_frag_count_return(page, drain_count))
453 + page_pool_return_page(pool, page);
456 +struct page *page_pool_alloc_frag(struct page_pool *pool,
457 + unsigned int *offset,
458 + unsigned int size, gfp_t gfp)
460 + unsigned int max_size = PAGE_SIZE << pool->p.order;
461 + struct page *page = pool->frag_page;
463 + if (WARN_ON(!(pool->p.flags & PP_FLAG_PAGE_FRAG) ||
467 + size = ALIGN(size, dma_get_cache_alignment());
468 + *offset = pool->frag_offset;
470 + if (page && *offset + size > max_size) {
471 + page = page_pool_drain_frag(pool, page);
477 + page = page_pool_alloc_pages(pool, gfp);
478 + if (unlikely(!page)) {
479 + pool->frag_page = NULL;
483 + pool->frag_page = page;
486 + pool->frag_users = 1;
488 + pool->frag_offset = size;
489 + page_pool_set_frag_count(page, BIAS_MAX);
493 + pool->frag_users++;
494 + pool->frag_offset = *offset + size;
497 +EXPORT_SYMBOL(page_pool_alloc_frag);
499 static void page_pool_empty_ring(struct page_pool *pool)
502 @@ -515,6 +617,8 @@ void page_pool_destroy(struct page_pool
503 if (!page_pool_put(pool))
506 + page_pool_free_frag(pool);
508 if (!page_pool_release(pool))
511 @@ -541,3 +645,32 @@ void page_pool_update_nid(struct page_po
514 EXPORT_SYMBOL(page_pool_update_nid);
516 +bool page_pool_return_skb_page(struct page *page)
518 + struct page_pool *pp;
520 + page = compound_head(page);
522 + /* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation
523 + * in order to preserve any existing bits, such as bit 0 for the
524 + * head page of compound page and bit 1 for pfmemalloc page, so
525 + * mask those bits for freeing side when doing below checking,
526 + * and page_is_pfmemalloc() is checked in __page_pool_put_page()
527 + * to avoid recycling the pfmemalloc page.
529 + if (unlikely((page->pp_magic & ~0x3UL) != PP_SIGNATURE))
534 + /* Driver set this to memory recycling info. Reset it on recycle.
535 + * This will *not* work for NIC using a split-page memory model.
536 + * The page will be returned to the pool here regardless of the
537 + * 'flipped' fragment being in use or not.
539 + page_pool_put_full_page(pp, page, false);
543 +EXPORT_SYMBOL(page_pool_return_skb_page);
544 --- a/include/linux/mm_types.h
545 +++ b/include/linux/mm_types.h
546 @@ -97,10 +97,25 @@ struct page {
548 struct { /* page_pool used by netstack */
550 - * @dma_addr: might require a 64-bit value on
551 - * 32-bit architectures.
552 + * @pp_magic: magic value to avoid recycling non
553 + * page_pool allocated pages.
555 - unsigned long dma_addr[2];
556 + unsigned long pp_magic;
557 + struct page_pool *pp;
558 + unsigned long _pp_mapping_pad;
559 + unsigned long dma_addr;
562 + * dma_addr_upper: might require a 64-bit
563 + * value on 32-bit architectures.
565 + unsigned long dma_addr_upper;
567 + * For frag page support, not supported in
568 + * 32-bit architectures with 64-bit DMA.
570 + atomic_long_t pp_frag_count;
573 struct { /* slab, slob and slub */
575 --- a/net/core/skbuff.c
576 +++ b/net/core/skbuff.c
577 @@ -594,13 +594,22 @@ static void skb_clone_fraglist(struct sk
581 +static bool skb_pp_recycle(struct sk_buff *skb, void *data)
583 + if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
585 + return page_pool_return_skb_page(virt_to_page(data));
588 static void skb_free_head(struct sk_buff *skb)
590 unsigned char *head = skb->head;
592 - if (skb->head_frag)
593 + if (skb->head_frag) {
594 + if (skb_pp_recycle(skb, head))
602 @@ -612,16 +621,27 @@ static void skb_release_data(struct sk_b
604 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
609 for (i = 0; i < shinfo->nr_frags; i++)
610 - __skb_frag_unref(&shinfo->frags[i]);
611 + __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle);
613 if (shinfo->frag_list)
614 kfree_skb_list(shinfo->frag_list);
616 skb_zcopy_clear(skb, true);
619 + /* When we clone an SKB we copy the reycling bit. The pp_recycle
620 + * bit is only set on the head though, so in order to avoid races
621 + * while trying to recycle fragments on __skb_frag_unref() we need
622 + * to make one SKB responsible for triggering the recycle path.
623 + * So disable the recycling bit if an SKB is cloned and we have
624 + * additional references to to the fragmented part of the SKB.
625 + * Eventually the last SKB will have the recycling bit set and it's
626 + * dataref set to 0, which will trigger the recycling
628 + skb->pp_recycle = 0;
632 @@ -1003,6 +1023,7 @@ static struct sk_buff *__skb_clone(struc
637 n->destructor = NULL;
640 @@ -3421,7 +3442,7 @@ int skb_shift(struct sk_buff *tgt, struc
641 fragto = &skb_shinfo(tgt)->frags[merge];
643 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
644 - __skb_frag_unref(fragfrom);
645 + __skb_frag_unref(fragfrom, skb->pp_recycle);
648 /* Reposition in the original skb */
649 @@ -5188,6 +5209,20 @@ bool skb_try_coalesce(struct sk_buff *to
653 + /* In general, avoid mixing slab allocated and page_pool allocated
654 + * pages within the same SKB. However when @to is not pp_recycle and
655 + * @from is cloned, we can transition frag pages from page_pool to
656 + * reference counted.
658 + * On the other hand, don't allow coalescing two pp_recycle SKBs if
659 + * @from is cloned, in case the SKB is using page_pool fragment
660 + * references (PP_FLAG_PAGE_FRAG). Since we only take full page
661 + * references for cloned SKBs at the moment that would result in
662 + * inconsistent reference counts.
664 + if (to->pp_recycle != (from->pp_recycle && !skb_cloned(from)))
667 if (len <= skb_tailroom(to)) {
669 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
670 --- a/include/linux/skbuff.h
671 +++ b/include/linux/skbuff.h
673 #include <linux/in6.h>
674 #include <linux/if_packet.h>
675 #include <net/flow.h>
676 +#include <net/page_pool.h>
677 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
678 #include <linux/netfilter/nf_conntrack_common.h>
680 @@ -786,7 +787,8 @@ struct sk_buff {
686 + pp_recycle:1; /* page_pool recycle indicator */
687 #ifdef CONFIG_SKB_EXTENSIONS
688 __u8 active_extensions;
690 @@ -3029,9 +3031,15 @@ static inline void skb_frag_ref(struct s
692 * Releases a reference on the paged fragment @frag.
694 -static inline void __skb_frag_unref(skb_frag_t *frag)
695 +static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
697 - put_page(skb_frag_page(frag));
698 + struct page *page = skb_frag_page(frag);
700 +#ifdef CONFIG_PAGE_POOL
701 + if (recycle && page_pool_return_skb_page(page))
708 @@ -3043,7 +3051,7 @@ static inline void __skb_frag_unref(skb_
710 static inline void skb_frag_unref(struct sk_buff *skb, int f)
712 - __skb_frag_unref(&skb_shinfo(skb)->frags[f]);
713 + __skb_frag_unref(&skb_shinfo(skb)->frags[f], skb->pp_recycle);
717 @@ -4642,5 +4650,12 @@ static inline u64 skb_get_kcov_handle(st
721 +#ifdef CONFIG_PAGE_POOL
722 +static inline void skb_mark_for_recycle(struct sk_buff *skb)
724 + skb->pp_recycle = 1;
728 #endif /* __KERNEL__ */
729 #endif /* _LINUX_SKBUFF_H */
730 --- a/drivers/net/ethernet/marvell/sky2.c
731 +++ b/drivers/net/ethernet/marvell/sky2.c
732 @@ -2501,7 +2501,7 @@ static void skb_put_frags(struct sk_buff
735 /* don't need this page */
736 - __skb_frag_unref(frag);
737 + __skb_frag_unref(frag, false);
738 --skb_shinfo(skb)->nr_frags;
740 size = min(length, (unsigned) PAGE_SIZE);
741 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
742 +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
743 @@ -526,7 +526,7 @@ static int mlx4_en_complete_rx_desc(stru
747 - __skb_frag_unref(skb_shinfo(skb)->frags + nr);
748 + __skb_frag_unref(skb_shinfo(skb)->frags + nr, false);
752 --- a/net/tls/tls_device.c
753 +++ b/net/tls/tls_device.c
754 @@ -131,7 +131,7 @@ static void destroy_record(struct tls_re
757 for (i = 0; i < record->num_frags; i++)
758 - __skb_frag_unref(&record->frags[i]);
759 + __skb_frag_unref(&record->frags[i], false);
763 --- a/include/linux/poison.h
764 +++ b/include/linux/poison.h
766 /********** security/ **********/
767 #define KEY_DESTROY 0xbd
769 +/********** net/core/page_pool.c **********/
770 +#define PP_SIGNATURE (0x40 + POISON_POINTER_DELTA)
773 --- a/include/linux/mm.h
774 +++ b/include/linux/mm.h
775 @@ -1602,7 +1602,7 @@ static inline bool page_is_pfmemalloc(st
776 * Page index cannot be this large so this must be
779 - return page->index == -1UL;
780 + return (uintptr_t)page->lru.next & BIT(1);
784 @@ -1611,12 +1611,12 @@ static inline bool page_is_pfmemalloc(st
786 static inline void set_page_pfmemalloc(struct page *page)
788 - page->index = -1UL;
789 + page->lru.next = (void *)BIT(1);
792 static inline void clear_page_pfmemalloc(struct page *page)
795 + page->lru.next = NULL;