target/linux/generic/backport-5.15/600-v5.18-page_pool-Add-allocation-stats.patch

   1 From 8610037e8106b48c79cfe0afb92b2b2466e51c3d Mon Sep 17 00:00:00 2001
   2 From: Joe Damato <jdamato@fastly.com>
   3 Date: Tue, 1 Mar 2022 23:55:47 -0800
   4 Subject: [PATCH] page_pool: Add allocation stats
   5
   6 Add per-pool statistics counters for the allocation path of a page pool.
   7 These stats are incremented in softirq context, so no locking or per-cpu
   8 variables are needed.
   9
  10 This code is disabled by default and a kernel config option is provided for
  11 users who wish to enable them.
  12
  13 The statistics added are:
  14         - fast: successful fast path allocations
  15         - slow: slow path order-0 allocations
  16         - slow_high_order: slow path high order allocations
  17         - empty: ptr ring is empty, so a slow path allocation was forced.
  18         - refill: an allocation which triggered a refill of the cache
  19         - waive: pages obtained from the ptr ring that cannot be added to
  20           the cache due to a NUMA mismatch.
  21
  22 Signed-off-by: Joe Damato <jdamato@fastly.com>
  23 Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
  24 Reviewed-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
  25 Signed-off-by: David S. Miller <davem@davemloft.net>
  26 ---
  27  include/net/page_pool.h | 18 ++++++++++++++++++
  28  net/Kconfig             | 13 +++++++++++++
  29  net/core/page_pool.c    | 24 ++++++++++++++++++++----
  30  3 files changed, 51 insertions(+), 4 deletions(-)
  31
  32 --- a/include/net/page_pool.h
  33 +++ b/include/net/page_pool.h
  34 @@ -82,6 +82,19 @@ struct page_pool_params {
  35         unsigned int    offset;  /* DMA addr offset */
  36  };
  37
  38 +#ifdef CONFIG_PAGE_POOL_STATS
  39 +struct page_pool_alloc_stats {
  40 +       u64 fast; /* fast path allocations */
  41 +       u64 slow; /* slow-path order 0 allocations */
  42 +       u64 slow_high_order; /* slow-path high order allocations */
  43 +       u64 empty; /* failed refills due to empty ptr ring, forcing
  44 +                   * slow path allocation
  45 +                   */
  46 +       u64 refill; /* allocations via successful refill */
  47 +       u64 waive;  /* failed refills due to numa zone mismatch */
  48 +};
  49 +#endif
  50 +
  51  struct page_pool {
  52         struct page_pool_params p;
  53
  54 @@ -132,6 +145,11 @@ struct page_pool {
  55         refcount_t user_cnt;
  56
  57         u64 destroy_cnt;
  58 +
  59 +#ifdef CONFIG_PAGE_POOL_STATS
  60 +       /* these stats are incremented while in softirq context */
  61 +       struct page_pool_alloc_stats alloc_stats;
  62 +#endif
  63  };
  64
  65  struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
  66 --- a/net/Kconfig
  67 +++ b/net/Kconfig
  68 @@ -434,6 +434,19 @@ config NET_DEVLINK
  69  config PAGE_POOL
  70         bool
  71
  72 +config PAGE_POOL_STATS
  73 +       default n
  74 +       bool "Page pool stats"
  75 +       depends on PAGE_POOL
  76 +       help
  77 +         Enable page pool statistics to track page allocation and recycling
  78 +         in page pools. This option incurs additional CPU cost in allocation
  79 +         and recycle paths and additional memory cost to store the statistics.
  80 +         These statistics are only available if this option is enabled and if
  81 +         the driver using the page pool supports exporting this data.
  82 +
  83 +         If unsure, say N.
  84 +
  85  config FAILOVER
  86         tristate "Generic failover module"
  87         help
  88 --- a/net/core/page_pool.c
  89 +++ b/net/core/page_pool.c
  90 @@ -26,6 +26,13 @@
  91
  92  #define BIAS_MAX       LONG_MAX
  93
  94 +#ifdef CONFIG_PAGE_POOL_STATS
  95 +/* alloc_stat_inc is intended to be used in softirq context */
  96 +#define alloc_stat_inc(pool, __stat)   (pool->alloc_stats.__stat++)
  97 +#else
  98 +#define alloc_stat_inc(pool, __stat)
  99 +#endif
 100 +
 101  static int page_pool_init(struct page_pool *pool,
 102                           const struct page_pool_params *params)
 103  {
 104 @@ -117,8 +124,10 @@ static struct page *page_pool_refill_all
 105         int pref_nid; /* preferred NUMA node */
 106
 107         /* Quicker fallback, avoid locks when ring is empty */
 108 -       if (__ptr_ring_empty(r))
 109 +       if (__ptr_ring_empty(r)) {
 110 +               alloc_stat_inc(pool, empty);
 111                 return NULL;
 112 +       }
 113
 114         /* Softirq guarantee CPU and thus NUMA node is stable. This,
 115          * assumes CPU refilling driver RX-ring will also run RX-NAPI.
 116 @@ -148,14 +157,17 @@ static struct page *page_pool_refill_all
 117                          * This limit stress on page buddy alloactor.
 118                          */
 119                         page_pool_return_page(pool, page);
 120 +                       alloc_stat_inc(pool, waive);
 121                         page = NULL;
 122                         break;
 123                 }
 124         } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);
 125
 126         /* Return last page */
 127 -       if (likely(pool->alloc.count > 0))
 128 +       if (likely(pool->alloc.count > 0)) {
 129                 page = pool->alloc.cache[--pool->alloc.count];
 130 +               alloc_stat_inc(pool, refill);
 131 +       }
 132
 133         spin_unlock(&r->consumer_lock);
 134         return page;
 135 @@ -170,6 +182,7 @@ static struct page *__page_pool_get_cach
 136         if (likely(pool->alloc.count)) {
 137                 /* Fast-path */
 138                 page = pool->alloc.cache[--pool->alloc.count];
 139 +               alloc_stat_inc(pool, fast);
 140         } else {
 141                 page = page_pool_refill_alloc_cache(pool);
 142         }
 143 @@ -241,6 +254,7 @@ static struct page *__page_pool_alloc_pa
 144                 return NULL;
 145         }
 146
 147 +       alloc_stat_inc(pool, slow_high_order);
 148         page_pool_set_pp_info(pool, page);
 149
 150         /* Track how many pages are held 'in-flight' */
 151 @@ -295,10 +309,12 @@ static struct page *__page_pool_alloc_pa
 152         }
 153
 154         /* Return last page */
 155 -       if (likely(pool->alloc.count > 0))
 156 +       if (likely(pool->alloc.count > 0)) {
 157                 page = pool->alloc.cache[--pool->alloc.count];
 158 -       else
 159 +               alloc_stat_inc(pool, slow);
 160 +       } else {
 161                 page = NULL;
 162 +       }
 163
 164         /* When page just alloc'ed is should/must have refcnt 1. */
 165         return page;