1 From fa6363828d314e837c5f79e97ea5e8c0d2f7f062 Mon Sep 17 00:00:00 2001
2 From: Yu Zhao <yuzhao@google.com>
3 Date: Wed, 21 Dec 2022 21:19:04 -0700
4 Subject: [PATCH 26/29] mm: multi-gen LRU: per-node lru_gen_page lists
6 For each node, memcgs are divided into two generations: the old and
7 the young. For each generation, memcgs are randomly sharded into
8 multiple bins to improve scalability. For each bin, an RCU hlist_nulls
9 is virtually divided into three segments: the head, the tail and the
12 An onlining memcg is added to the tail of a random bin in the old
13 generation. The eviction starts at the head of a random bin in the old
14 generation. The per-node memcg generation counter, whose reminder (mod
15 2) indexes the old generation, is incremented when all its bins become
18 There are four operations:
19 1. MEMCG_LRU_HEAD, which moves an memcg to the head of a random bin in
20 its current generation (old or young) and updates its "seg" to
22 2. MEMCG_LRU_TAIL, which moves an memcg to the tail of a random bin in
23 its current generation (old or young) and updates its "seg" to
25 3. MEMCG_LRU_OLD, which moves an memcg to the head of a random bin in
26 the old generation, updates its "gen" to "old" and resets its "seg"
28 4. MEMCG_LRU_YOUNG, which moves an memcg to the tail of a random bin
29 in the young generation, updates its "gen" to "young" and resets
30 its "seg" to "default".
32 The events that trigger the above operations are:
33 1. Exceeding the soft limit, which triggers MEMCG_LRU_HEAD;
34 2. The first attempt to reclaim an memcg below low, which triggers
36 3. The first attempt to reclaim an memcg below reclaimable size
37 threshold, which triggers MEMCG_LRU_TAIL;
38 4. The second attempt to reclaim an memcg below reclaimable size
39 threshold, which triggers MEMCG_LRU_YOUNG;
40 5. Attempting to reclaim an memcg below min, which triggers
42 6. Finishing the aging on the eviction path, which triggers
44 7. Offlining an memcg, which triggers MEMCG_LRU_OLD.
46 Note that memcg LRU only applies to global reclaim, and the
47 round-robin incrementing of their max_seq counters ensures the
48 eventual fairness to all eligible memcgs. For memcg reclaim, it still
49 relies on mem_cgroup_iter().
51 Link: https://lkml.kernel.org/r/20221222041905.2431096-7-yuzhao@google.com
52 Signed-off-by: Yu Zhao <yuzhao@google.com>
53 Cc: Johannes Weiner <hannes@cmpxchg.org>
54 Cc: Jonathan Corbet <corbet@lwn.net>
55 Cc: Michael Larabel <Michael@MichaelLarabel.com>
56 Cc: Michal Hocko <mhocko@kernel.org>
57 Cc: Mike Rapoport <rppt@kernel.org>
58 Cc: Roman Gushchin <roman.gushchin@linux.dev>
59 Cc: Suren Baghdasaryan <surenb@google.com>
60 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
62 include/linux/memcontrol.h | 10 +
63 include/linux/mm_inline.h | 17 ++
64 include/linux/mmzone.h | 117 +++++++++++-
65 mm/memcontrol.c | 16 ++
67 mm/vmscan.c | 373 +++++++++++++++++++++++++++++++++----
68 6 files changed, 499 insertions(+), 35 deletions(-)
70 diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
71 index 3736405cbcf6..2e405fd88846 100644
72 --- a/include/linux/memcontrol.h
73 +++ b/include/linux/memcontrol.h
74 @@ -818,6 +818,11 @@ static inline void obj_cgroup_put(struct obj_cgroup *objcg)
75 percpu_ref_put(&objcg->refcnt);
78 +static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg)
80 + return !memcg || css_tryget(&memcg->css);
83 static inline void mem_cgroup_put(struct mem_cgroup *memcg)
86 @@ -1283,6 +1288,11 @@ struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css)
90 +static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg)
95 static inline void mem_cgroup_put(struct mem_cgroup *memcg)
98 diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
99 index 4adc9ba59569..9138c2e638ce 100644
100 --- a/include/linux/mm_inline.h
101 +++ b/include/linux/mm_inline.h
102 @@ -112,6 +112,18 @@ static inline bool lru_gen_in_fault(void)
103 return current->in_lru_fault;
107 +static inline int lru_gen_memcg_seg(struct lruvec *lruvec)
109 + return READ_ONCE(lruvec->lrugen.seg);
112 +static inline int lru_gen_memcg_seg(struct lruvec *lruvec)
118 static inline int lru_gen_from_seq(unsigned long seq)
120 return seq % MAX_NR_GENS;
121 @@ -287,6 +299,11 @@ static inline bool lru_gen_in_fault(void)
125 +static inline int lru_gen_memcg_seg(struct lruvec *lruvec)
130 static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bool reclaiming)
133 diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
134 index 7b8a26aaf381..4bbf191517e2 100644
135 --- a/include/linux/mmzone.h
136 +++ b/include/linux/mmzone.h
139 #include <linux/spinlock.h>
140 #include <linux/list.h>
141 +#include <linux/list_nulls.h>
142 #include <linux/wait.h>
143 #include <linux/bitops.h>
144 #include <linux/cache.h>
145 @@ -357,6 +358,15 @@ struct page_vma_mapped_walk;
146 #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
147 #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
149 +/* see the comment on MEMCG_NR_GENS */
158 #ifdef CONFIG_LRU_GEN
161 @@ -416,6 +426,14 @@ struct lru_gen_page {
162 atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
163 /* whether the multi-gen LRU is enabled */
166 + /* the memcg generation this lru_gen_page belongs to */
168 + /* the list segment this lru_gen_page belongs to */
170 + /* per-node lru_gen_page list for global reclaim */
171 + struct hlist_nulls_node list;
176 @@ -469,12 +487,87 @@ void lru_gen_init_lruvec(struct lruvec *lruvec);
177 void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
182 + * For each node, memcgs are divided into two generations: the old and the
183 + * young. For each generation, memcgs are randomly sharded into multiple bins
184 + * to improve scalability. For each bin, the hlist_nulls is virtually divided
185 + * into three segments: the head, the tail and the default.
187 + * An onlining memcg is added to the tail of a random bin in the old generation.
188 + * The eviction starts at the head of a random bin in the old generation. The
189 + * per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes
190 + * the old generation, is incremented when all its bins become empty.
192 + * There are four operations:
193 + * 1. MEMCG_LRU_HEAD, which moves an memcg to the head of a random bin in its
194 + * current generation (old or young) and updates its "seg" to "head";
195 + * 2. MEMCG_LRU_TAIL, which moves an memcg to the tail of a random bin in its
196 + * current generation (old or young) and updates its "seg" to "tail";
197 + * 3. MEMCG_LRU_OLD, which moves an memcg to the head of a random bin in the old
198 + * generation, updates its "gen" to "old" and resets its "seg" to "default";
199 + * 4. MEMCG_LRU_YOUNG, which moves an memcg to the tail of a random bin in the
200 + * young generation, updates its "gen" to "young" and resets its "seg" to
203 + * The events that trigger the above operations are:
204 + * 1. Exceeding the soft limit, which triggers MEMCG_LRU_HEAD;
205 + * 2. The first attempt to reclaim an memcg below low, which triggers
207 + * 3. The first attempt to reclaim an memcg below reclaimable size threshold,
208 + * which triggers MEMCG_LRU_TAIL;
209 + * 4. The second attempt to reclaim an memcg below reclaimable size threshold,
210 + * which triggers MEMCG_LRU_YOUNG;
211 + * 5. Attempting to reclaim an memcg below min, which triggers MEMCG_LRU_YOUNG;
212 + * 6. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG;
213 + * 7. Offlining an memcg, which triggers MEMCG_LRU_OLD.
215 + * Note that memcg LRU only applies to global reclaim, and the round-robin
216 + * incrementing of their max_seq counters ensures the eventual fairness to all
217 + * eligible memcgs. For memcg reclaim, it still relies on mem_cgroup_iter().
219 +#define MEMCG_NR_GENS 2
220 +#define MEMCG_NR_BINS 8
222 +struct lru_gen_memcg {
223 + /* the per-node memcg generation counter */
225 + /* each memcg has one lru_gen_page per node */
226 + unsigned long nr_memcgs[MEMCG_NR_GENS];
227 + /* per-node lru_gen_page list for global reclaim */
228 + struct hlist_nulls_head fifo[MEMCG_NR_GENS][MEMCG_NR_BINS];
229 + /* protects the above */
233 +void lru_gen_init_pgdat(struct pglist_data *pgdat);
235 void lru_gen_init_memcg(struct mem_cgroup *memcg);
236 void lru_gen_exit_memcg(struct mem_cgroup *memcg);
238 +void lru_gen_online_memcg(struct mem_cgroup *memcg);
239 +void lru_gen_offline_memcg(struct mem_cgroup *memcg);
240 +void lru_gen_release_memcg(struct mem_cgroup *memcg);
241 +void lru_gen_rotate_memcg(struct lruvec *lruvec, int op);
243 +#else /* !CONFIG_MEMCG */
245 +#define MEMCG_NR_GENS 1
247 +struct lru_gen_memcg {
250 +static inline void lru_gen_init_pgdat(struct pglist_data *pgdat)
254 +#endif /* CONFIG_MEMCG */
256 #else /* !CONFIG_LRU_GEN */
258 +static inline void lru_gen_init_pgdat(struct pglist_data *pgdat)
262 static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
265 @@ -484,6 +577,7 @@ static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
270 static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
273 @@ -491,7 +585,24 @@ static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
274 static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
279 +static inline void lru_gen_online_memcg(struct mem_cgroup *memcg)
283 +static inline void lru_gen_offline_memcg(struct mem_cgroup *memcg)
287 +static inline void lru_gen_release_memcg(struct mem_cgroup *memcg)
291 +static inline void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
295 +#endif /* CONFIG_MEMCG */
297 #endif /* CONFIG_LRU_GEN */
299 @@ -1105,6 +1216,8 @@ typedef struct pglist_data {
300 #ifdef CONFIG_LRU_GEN
301 /* kswap mm walk data */
302 struct lru_gen_mm_walk mm_walk;
303 + /* lru_gen_page list */
304 + struct lru_gen_memcg memcg_lru;
308 diff --git a/mm/memcontrol.c b/mm/memcontrol.c
309 index ed87d1256f0e..172adfbee06e 100644
310 --- a/mm/memcontrol.c
311 +++ b/mm/memcontrol.c
312 @@ -549,6 +549,16 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
313 struct mem_cgroup_per_node *mz;
314 struct mem_cgroup_tree_per_node *mctz;
316 + if (lru_gen_enabled()) {
317 + struct lruvec *lruvec = &mem_cgroup_page_nodeinfo(memcg, page)->lruvec;
319 + /* see the comment on MEMCG_NR_GENS */
320 + if (soft_limit_excess(memcg) && lru_gen_memcg_seg(lruvec) != MEMCG_LRU_HEAD)
321 + lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD);
326 mctz = soft_limit_tree_from_page(page);
329 @@ -3433,6 +3443,9 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
330 unsigned long excess;
331 unsigned long nr_scanned;
333 + if (lru_gen_enabled())
339 @@ -5321,6 +5334,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
340 if (unlikely(mem_cgroup_is_root(memcg)))
341 queue_delayed_work(system_unbound_wq, &stats_flush_dwork,
343 + lru_gen_online_memcg(memcg);
347 @@ -5347,6 +5361,7 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
348 memcg_offline_kmem(memcg);
349 reparent_shrinker_deferred(memcg);
350 wb_memcg_offline(memcg);
351 + lru_gen_offline_memcg(memcg);
353 drain_all_stock(memcg);
355 @@ -5358,6 +5373,7 @@ static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
356 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
358 invalidate_reclaim_iterators(memcg);
359 + lru_gen_release_memcg(memcg);
362 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
363 diff --git a/mm/page_alloc.c b/mm/page_alloc.c
364 index c929357fbefe..6459d9c018be 100644
365 --- a/mm/page_alloc.c
366 +++ b/mm/page_alloc.c
367 @@ -7645,6 +7645,7 @@ static void __init free_area_init_node(int nid)
368 pgdat_set_deferred_range(pgdat);
370 free_area_init_core(pgdat);
371 + lru_gen_init_pgdat(pgdat);
374 void __init free_area_init_memoryless_node(int nid)
375 diff --git a/mm/vmscan.c b/mm/vmscan.c
376 index cb026e2714d7..3d8e0665186c 100644
380 #include <linux/shmem_fs.h>
381 #include <linux/ctype.h>
382 #include <linux/debugfs.h>
383 +#include <linux/rculist_nulls.h>
384 +#include <linux/random.h>
386 #include <asm/tlbflush.h>
387 #include <asm/div64.h>
388 @@ -129,11 +131,6 @@ struct scan_control {
389 /* Always discard instead of demoting to lower tier memory */
390 unsigned int no_demotion:1;
392 -#ifdef CONFIG_LRU_GEN
393 - /* help kswapd make better choices among multiple memcgs */
394 - unsigned long last_reclaimed;
397 /* Allocation order */
400 @@ -2880,6 +2877,9 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS);
401 for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
402 for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
404 +#define get_memcg_gen(seq) ((seq) % MEMCG_NR_GENS)
405 +#define get_memcg_bin(bin) ((bin) % MEMCG_NR_BINS)
407 static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
409 struct pglist_data *pgdat = NODE_DATA(nid);
410 @@ -4169,8 +4169,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
411 if (sc->priority <= DEF_PRIORITY - 2)
412 wait_event_killable(lruvec->mm_state.wait,
413 max_seq < READ_ONCE(lrugen->max_seq));
415 - return max_seq < READ_ONCE(lrugen->max_seq);
419 VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq));
420 @@ -4243,8 +4242,6 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
422 VM_WARN_ON_ONCE(!current_is_kswapd());
424 - sc->last_reclaimed = sc->nr_reclaimed;
426 /* check the order to exclude compaction-induced reclaim */
427 if (!min_ttl || sc->order || sc->priority == DEF_PRIORITY)
429 @@ -4833,8 +4830,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
430 * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
433 -static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
435 +static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap)
437 unsigned long nr_to_scan;
438 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
439 @@ -4851,10 +4847,8 @@ static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *
440 if (sc->priority == DEF_PRIORITY)
443 - try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false);
445 /* skip this lruvec as it's low on cold pages */
447 + return try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false) ? -1 : 0;
450 static unsigned long get_nr_to_reclaim(struct scan_control *sc)
451 @@ -4863,29 +4857,18 @@ static unsigned long get_nr_to_reclaim(struct scan_control *sc)
452 if (!global_reclaim(sc))
455 - /* discount the previous progress for kswapd */
456 - if (current_is_kswapd())
457 - return sc->nr_to_reclaim + sc->last_reclaimed;
459 return max(sc->nr_to_reclaim, compact_gap(sc->order));
462 -static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
463 +static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
465 - struct blk_plug plug;
467 unsigned long scanned = 0;
468 unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
472 - blk_start_plug(&plug);
474 - set_mm_walk(lruvec_pgdat(lruvec));
479 - unsigned long nr_to_scan;
482 swappiness = get_swappiness(lruvec, sc);
483 @@ -4895,7 +4878,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
486 nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
488 + if (nr_to_scan <= 0)
491 delta = evict_pages(lruvec, sc, swappiness);
492 @@ -4912,11 +4895,251 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
496 + /* whether try_to_inc_max_seq() was successful */
497 + return nr_to_scan < 0;
500 +static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
503 + unsigned long scanned = sc->nr_scanned;
504 + unsigned long reclaimed = sc->nr_reclaimed;
505 + int seg = lru_gen_memcg_seg(lruvec);
506 + struct mem_cgroup *memcg = lruvec_memcg(lruvec);
507 + struct pglist_data *pgdat = lruvec_pgdat(lruvec);
509 + /* see the comment on MEMCG_NR_GENS */
510 + if (!lruvec_is_sizable(lruvec, sc))
511 + return seg != MEMCG_LRU_TAIL ? MEMCG_LRU_TAIL : MEMCG_LRU_YOUNG;
513 + mem_cgroup_calculate_protection(NULL, memcg);
515 + if (mem_cgroup_below_min(memcg))
516 + return MEMCG_LRU_YOUNG;
518 + if (mem_cgroup_below_low(memcg)) {
519 + /* see the comment on MEMCG_NR_GENS */
520 + if (seg != MEMCG_LRU_TAIL)
521 + return MEMCG_LRU_TAIL;
523 + memcg_memory_event(memcg, MEMCG_LOW);
526 + success = try_to_shrink_lruvec(lruvec, sc);
528 + shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);
530 + vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned,
531 + sc->nr_reclaimed - reclaimed);
533 + sc->nr_reclaimed += current->reclaim_state->reclaimed_slab;
534 + current->reclaim_state->reclaimed_slab = 0;
536 + return success ? MEMCG_LRU_YOUNG : 0;
541 +static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
546 + struct lruvec *lruvec;
547 + struct lru_gen_page *lrugen;
548 + const struct hlist_nulls_node *pos;
550 + struct mem_cgroup *memcg = NULL;
551 + unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
553 + bin = first_bin = prandom_u32_max(MEMCG_NR_BINS);
555 + gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq));
559 + hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) {
561 + lru_gen_rotate_memcg(lruvec, op);
563 + mem_cgroup_put(memcg);
565 + lruvec = container_of(lrugen, struct lruvec, lrugen);
566 + memcg = lruvec_memcg(lruvec);
568 + if (!mem_cgroup_tryget(memcg)) {
576 + op = shrink_one(lruvec, sc);
578 + if (sc->nr_reclaimed >= nr_to_reclaim)
586 + /* restart if raced with lru_gen_rotate_memcg() */
587 + if (gen != get_nulls_value(pos))
590 + /* try the rest of the bins of the current generation */
591 + bin = get_memcg_bin(bin + 1);
592 + if (bin != first_bin)
596 + lru_gen_rotate_memcg(lruvec, op);
598 + mem_cgroup_put(memcg);
601 +static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
603 + struct blk_plug plug;
605 + VM_WARN_ON_ONCE(global_reclaim(sc));
609 + blk_start_plug(&plug);
611 + set_mm_walk(lruvec_pgdat(lruvec));
613 + if (try_to_shrink_lruvec(lruvec, sc))
614 + lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
618 + blk_finish_plug(&plug);
621 +#else /* !CONFIG_MEMCG */
623 +static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
628 +static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
635 +static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc)
638 + unsigned long reclaimable;
639 + struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
641 + if (sc->priority != DEF_PRIORITY || sc->nr_to_reclaim < MIN_LRU_BATCH)
644 + * Determine the initial priority based on ((total / MEMCG_NR_GENS) >>
645 + * priority) * reclaimed_to_scanned_ratio = nr_to_reclaim, where the
646 + * estimated reclaimed_to_scanned_ratio = inactive / total.
648 + reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE);
649 + if (get_swappiness(lruvec, sc))
650 + reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON);
652 + reclaimable /= MEMCG_NR_GENS;
654 + /* round down reclaimable and round up sc->nr_to_reclaim */
655 + priority = fls_long(reclaimable) - 1 - fls_long(sc->nr_to_reclaim - 1);
657 + sc->priority = clamp(priority, 0, DEF_PRIORITY);
660 +static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)
662 + struct blk_plug plug;
663 + unsigned long reclaimed = sc->nr_reclaimed;
665 + VM_WARN_ON_ONCE(!global_reclaim(sc));
669 + blk_start_plug(&plug);
671 + set_mm_walk(pgdat);
673 + set_initial_priority(pgdat, sc);
675 + if (current_is_kswapd())
676 + sc->nr_reclaimed = 0;
678 + if (mem_cgroup_disabled())
679 + shrink_one(&pgdat->__lruvec, sc);
681 + shrink_many(pgdat, sc);
683 + if (current_is_kswapd())
684 + sc->nr_reclaimed += reclaimed;
688 blk_finish_plug(&plug);
690 + /* kswapd should never fail */
691 + pgdat->kswapd_failures = 0;
695 +void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
699 + int bin = prandom_u32_max(MEMCG_NR_BINS);
700 + struct pglist_data *pgdat = lruvec_pgdat(lruvec);
702 + spin_lock(&pgdat->memcg_lru.lock);
704 + VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
707 + new = old = lruvec->lrugen.gen;
709 + /* see the comment on MEMCG_NR_GENS */
710 + if (op == MEMCG_LRU_HEAD)
711 + seg = MEMCG_LRU_HEAD;
712 + else if (op == MEMCG_LRU_TAIL)
713 + seg = MEMCG_LRU_TAIL;
714 + else if (op == MEMCG_LRU_OLD)
715 + new = get_memcg_gen(pgdat->memcg_lru.seq);
716 + else if (op == MEMCG_LRU_YOUNG)
717 + new = get_memcg_gen(pgdat->memcg_lru.seq + 1);
719 + VM_WARN_ON_ONCE(true);
721 + hlist_nulls_del_rcu(&lruvec->lrugen.list);
723 + if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD)
724 + hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
726 + hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
728 + pgdat->memcg_lru.nr_memcgs[old]--;
729 + pgdat->memcg_lru.nr_memcgs[new]++;
731 + lruvec->lrugen.gen = new;
732 + WRITE_ONCE(lruvec->lrugen.seg, seg);
734 + if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq))
735 + WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
737 + spin_unlock(&pgdat->memcg_lru.lock);
741 /******************************************************************************
743 ******************************************************************************/
744 @@ -5370,11 +5593,11 @@ static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq,
746 if (!mem_cgroup_disabled()) {
749 memcg = mem_cgroup_from_id(memcg_id);
751 - if (memcg && !css_tryget(&memcg->css))
752 + if (!mem_cgroup_tryget(memcg))
759 @@ -5521,6 +5744,19 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
764 +void lru_gen_init_pgdat(struct pglist_data *pgdat)
768 + spin_lock_init(&pgdat->memcg_lru.lock);
770 + for (i = 0; i < MEMCG_NR_GENS; i++) {
771 + for (j = 0; j < MEMCG_NR_BINS; j++)
772 + INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i);
776 void lru_gen_init_memcg(struct mem_cgroup *memcg)
778 INIT_LIST_HEAD(&memcg->mm_list.fifo);
779 @@ -5544,7 +5780,69 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
785 +void lru_gen_online_memcg(struct mem_cgroup *memcg)
789 + int bin = prandom_u32_max(MEMCG_NR_BINS);
791 + for_each_node(nid) {
792 + struct pglist_data *pgdat = NODE_DATA(nid);
793 + struct lruvec *lruvec = get_lruvec(memcg, nid);
795 + spin_lock(&pgdat->memcg_lru.lock);
797 + VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
799 + gen = get_memcg_gen(pgdat->memcg_lru.seq);
801 + hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
802 + pgdat->memcg_lru.nr_memcgs[gen]++;
804 + lruvec->lrugen.gen = gen;
806 + spin_unlock(&pgdat->memcg_lru.lock);
810 +void lru_gen_offline_memcg(struct mem_cgroup *memcg)
814 + for_each_node(nid) {
815 + struct lruvec *lruvec = get_lruvec(memcg, nid);
817 + lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD);
821 +void lru_gen_release_memcg(struct mem_cgroup *memcg)
826 + for_each_node(nid) {
827 + struct pglist_data *pgdat = NODE_DATA(nid);
828 + struct lruvec *lruvec = get_lruvec(memcg, nid);
830 + spin_lock(&pgdat->memcg_lru.lock);
832 + VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
834 + gen = lruvec->lrugen.gen;
836 + hlist_nulls_del_rcu(&lruvec->lrugen.list);
837 + pgdat->memcg_lru.nr_memcgs[gen]--;
839 + if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq))
840 + WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
842 + spin_unlock(&pgdat->memcg_lru.lock);
846 +#endif /* CONFIG_MEMCG */
848 static int __init init_lru_gen(void)
850 @@ -5571,6 +5869,10 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
854 +static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)
858 #endif /* CONFIG_LRU_GEN */
860 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
861 @@ -5584,7 +5886,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
862 bool proportional_reclaim;
863 struct blk_plug plug;
865 - if (lru_gen_enabled()) {
866 + if (lru_gen_enabled() && !global_reclaim(sc)) {
867 lru_gen_shrink_lruvec(lruvec, sc);
870 @@ -5826,6 +6128,11 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
871 struct lruvec *target_lruvec;
872 bool reclaimable = false;
874 + if (lru_gen_enabled() && global_reclaim(sc)) {
875 + lru_gen_shrink_node(pgdat, sc);
879 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);