-commit 8610037e8106b48c79cfe0afb92b2b2466e51c3d
-Author: Joe Damato <jdamato@fastly.com>
-Date: Tue Mar 1 23:55:47 2022 -0800
+From 8610037e8106b48c79cfe0afb92b2b2466e51c3d Mon Sep 17 00:00:00 2001
+From: Joe Damato <jdamato@fastly.com>
+Date: Tue, 1 Mar 2022 23:55:47 -0800
+Subject: [PATCH] page_pool: Add allocation stats
- page_pool: Add allocation stats
-
- Add per-pool statistics counters for the allocation path of a page pool.
- These stats are incremented in softirq context, so no locking or per-cpu
- variables are needed.
-
- This code is disabled by default and a kernel config option is provided for
- users who wish to enable them.
-
- The statistics added are:
- - fast: successful fast path allocations
- - slow: slow path order-0 allocations
- - slow_high_order: slow path high order allocations
- - empty: ptr ring is empty, so a slow path allocation was forced.
- - refill: an allocation which triggered a refill of the cache
- - waive: pages obtained from the ptr ring that cannot be added to
- the cache due to a NUMA mismatch.
-
- Signed-off-by: Joe Damato <jdamato@fastly.com>
- Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
- Reviewed-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
- Signed-off-by: David S. Miller <davem@davemloft.net>
+Add per-pool statistics counters for the allocation path of a page pool.
+These stats are incremented in softirq context, so no locking or per-cpu
+variables are needed.
+
+This code is disabled by default and a kernel config option is provided for
+users who wish to enable them.
+
+The statistics added are:
+ - fast: successful fast path allocations
+ - slow: slow path order-0 allocations
+ - slow_high_order: slow path high order allocations
+ - empty: ptr ring is empty, so a slow path allocation was forced.
+ - refill: an allocation which triggered a refill of the cache
+ - waive: pages obtained from the ptr ring that cannot be added to
+ the cache due to a NUMA mismatch.
+
+Signed-off-by: Joe Damato <jdamato@fastly.com>
+Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
+Reviewed-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ include/net/page_pool.h | 18 ++++++++++++++++++
+ net/Kconfig | 13 +++++++++++++
+ net/core/page_pool.c | 24 ++++++++++++++++++++----
+ 3 files changed, 51 insertions(+), 4 deletions(-)
--- a/include/net/page_pool.h
+++ b/include/net/page_pool.h
struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
--- a/net/Kconfig
+++ b/net/Kconfig
-@@ -434,6 +434,19 @@ config NET_DEVLINK
+@@ -432,6 +432,19 @@ config NET_DEVLINK
config PAGE_POOL
bool
help
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
-@@ -26,6 +26,13 @@
-
- #define BIAS_MAX LONG_MAX
+@@ -49,6 +49,13 @@ static void page_pool_producer_unlock(st
+ spin_unlock_bh(&pool->ring.producer_lock);
+ }
+#ifdef CONFIG_PAGE_POOL_STATS
+/* alloc_stat_inc is intended to be used in softirq context */
static int page_pool_init(struct page_pool *pool,
const struct page_pool_params *params)
{
-@@ -117,8 +124,10 @@ static struct page *page_pool_refill_all
+@@ -140,8 +147,10 @@ static struct page *page_pool_refill_all
int pref_nid; /* preferred NUMA node */
/* Quicker fallback, avoid locks when ring is empty */
/* Softirq guarantee CPU and thus NUMA node is stable. This,
* assumes CPU refilling driver RX-ring will also run RX-NAPI.
-@@ -148,14 +157,17 @@ static struct page *page_pool_refill_all
+@@ -171,14 +180,17 @@ static struct page *page_pool_refill_all
* This limit stress on page buddy alloactor.
*/
page_pool_return_page(pool, page);
spin_unlock(&r->consumer_lock);
return page;
-@@ -170,6 +182,7 @@ static struct page *__page_pool_get_cach
+@@ -193,6 +205,7 @@ static struct page *__page_pool_get_cach
if (likely(pool->alloc.count)) {
/* Fast-path */
page = pool->alloc.cache[--pool->alloc.count];
} else {
page = page_pool_refill_alloc_cache(pool);
}
-@@ -241,6 +254,7 @@ static struct page *__page_pool_alloc_pa
+@@ -264,6 +277,7 @@ static struct page *__page_pool_alloc_pa
return NULL;
}
page_pool_set_pp_info(pool, page);
/* Track how many pages are held 'in-flight' */
-@@ -295,10 +309,12 @@ static struct page *__page_pool_alloc_pa
+@@ -318,10 +332,12 @@ static struct page *__page_pool_alloc_pa
}
/* Return last page */