mm/vmscan.c | 373 +++++++++++++++++++++++++++++++++----
6 files changed, 499 insertions(+), 35 deletions(-)
-diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
-index 3736405cbcf6..2e405fd88846 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
-@@ -818,6 +818,11 @@ static inline void obj_cgroup_put(struct obj_cgroup *objcg)
+@@ -818,6 +818,11 @@ static inline void obj_cgroup_put(struct
percpu_ref_put(&objcg->refcnt);
}
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
if (memcg)
-@@ -1283,6 +1288,11 @@ struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css)
+@@ -1283,6 +1288,11 @@ struct mem_cgroup *mem_cgroup_from_css(s
return NULL;
}
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index 4adc9ba59569..9138c2e638ce 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -112,6 +112,18 @@ static inline bool lru_gen_in_fault(void)
+@@ -112,6 +112,18 @@ static inline bool lru_gen_in_fault(void
return current->in_lru_fault;
}
static inline int lru_gen_from_seq(unsigned long seq)
{
return seq % MAX_NR_GENS;
-@@ -287,6 +299,11 @@ static inline bool lru_gen_in_fault(void)
+@@ -287,6 +299,11 @@ static inline bool lru_gen_in_fault(void
return false;
}
static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bool reclaiming)
{
return false;
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 7b8a26aaf381..4bbf191517e2 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -7,6 +7,7 @@
};
enum {
-@@ -469,12 +487,87 @@ void lru_gen_init_lruvec(struct lruvec *lruvec);
+@@ -469,12 +487,87 @@ void lru_gen_init_lruvec(struct lruvec *
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
#ifdef CONFIG_MEMCG
static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
{
}
-@@ -484,6 +577,7 @@ static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -484,6 +577,7 @@ static inline void lru_gen_look_around(s
}
#ifdef CONFIG_MEMCG
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
}
-@@ -491,7 +585,24 @@ static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
+@@ -491,7 +585,24 @@ static inline void lru_gen_init_memcg(st
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
}
#endif
ZONE_PADDING(_pad2_)
-diff --git a/mm/memcontrol.c b/mm/memcontrol.c
-index ed87d1256f0e..172adfbee06e 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
-@@ -549,6 +549,16 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
+@@ -549,6 +549,16 @@ static void mem_cgroup_update_tree(struc
struct mem_cgroup_per_node *mz;
struct mem_cgroup_tree_per_node *mctz;
mctz = soft_limit_tree_from_page(page);
if (!mctz)
return;
-@@ -3433,6 +3443,9 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
+@@ -3433,6 +3443,9 @@ unsigned long mem_cgroup_soft_limit_recl
unsigned long excess;
unsigned long nr_scanned;
if (order > 0)
return 0;
-@@ -5321,6 +5334,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
+@@ -5321,6 +5334,7 @@ static int mem_cgroup_css_online(struct
if (unlikely(mem_cgroup_is_root(memcg)))
queue_delayed_work(system_unbound_wq, &stats_flush_dwork,
2UL*HZ);
return 0;
}
-@@ -5347,6 +5361,7 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
+@@ -5347,6 +5361,7 @@ static void mem_cgroup_css_offline(struc
memcg_offline_kmem(memcg);
reparent_shrinker_deferred(memcg);
wb_memcg_offline(memcg);
drain_all_stock(memcg);
-@@ -5358,6 +5373,7 @@ static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
+@@ -5358,6 +5373,7 @@ static void mem_cgroup_css_released(stru
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
invalidate_reclaim_iterators(memcg);
}
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
-diff --git a/mm/page_alloc.c b/mm/page_alloc.c
-index c929357fbefe..6459d9c018be 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
-@@ -7645,6 +7645,7 @@ static void __init free_area_init_node(int nid)
+@@ -7661,6 +7661,7 @@ static void __init free_area_init_node(i
pgdat_set_deferred_range(pgdat);
free_area_init_core(pgdat);
}
void __init free_area_init_memoryless_node(int nid)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index cb026e2714d7..3d8e0665186c 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -54,6 +54,8 @@
/* Allocation order */
s8 order;
-@@ -2880,6 +2877,9 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS);
+@@ -2880,6 +2877,9 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_ca
for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
{
struct pglist_data *pgdat = NODE_DATA(nid);
-@@ -4169,8 +4169,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4169,8 +4169,7 @@ done:
if (sc->priority <= DEF_PRIORITY - 2)
wait_event_killable(lruvec->mm_state.wait,
max_seq < READ_ONCE(lrugen->max_seq));
}
VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq));
-@@ -4243,8 +4242,6 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -4243,8 +4242,6 @@ static void lru_gen_age_node(struct pgli
VM_WARN_ON_ONCE(!current_is_kswapd());
/* check the order to exclude compaction-induced reclaim */
if (!min_ttl || sc->order || sc->priority == DEF_PRIORITY)
return;
-@@ -4833,8 +4830,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4833,8 +4830,7 @@ static bool should_run_aging(struct lruv
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
* reclaim.
*/
{
unsigned long nr_to_scan;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
-@@ -4851,10 +4847,8 @@ static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *
+@@ -4851,10 +4847,8 @@ static unsigned long get_nr_to_scan(stru
if (sc->priority == DEF_PRIORITY)
return nr_to_scan;
}
static unsigned long get_nr_to_reclaim(struct scan_control *sc)
-@@ -4863,29 +4857,18 @@ static unsigned long get_nr_to_reclaim(struct scan_control *sc)
+@@ -4863,29 +4857,18 @@ static unsigned long get_nr_to_reclaim(s
if (!global_reclaim(sc))
return -1;
if (sc->may_swap)
swappiness = get_swappiness(lruvec, sc);
-@@ -4895,7 +4878,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -4895,7 +4878,7 @@ static void lru_gen_shrink_lruvec(struct
swappiness = 0;
nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
break;
delta = evict_pages(lruvec, sc, swappiness);
-@@ -4912,11 +4895,251 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -4912,10 +4895,250 @@ static void lru_gen_shrink_lruvec(struct
cond_resched();
}
+
+ /* kswapd should never fail */
+ pgdat->kswapd_failures = 0;
- }
-
++}
++
+#ifdef CONFIG_MEMCG
+void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
+{
+ WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
+
+ spin_unlock(&pgdat->memcg_lru.lock);
-+}
+ }
+#endif
-+
+
/******************************************************************************
* state change
- ******************************************************************************/
-@@ -5370,11 +5593,11 @@ static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq,
+@@ -5370,11 +5593,11 @@ static int run_cmd(char cmd, int memcg_i
if (!mem_cgroup_disabled()) {
rcu_read_lock();
rcu_read_unlock();
if (!memcg)
-@@ -5521,6 +5744,19 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -5521,6 +5744,19 @@ void lru_gen_init_lruvec(struct lruvec *
}
#ifdef CONFIG_MEMCG
void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
INIT_LIST_HEAD(&memcg->mm_list.fifo);
-@@ -5544,7 +5780,69 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -5544,7 +5780,69 @@ void lru_gen_exit_memcg(struct mem_cgrou
}
}
}
static int __init init_lru_gen(void)
{
-@@ -5571,6 +5869,10 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5571,6 +5869,10 @@ static void lru_gen_shrink_lruvec(struct
{
}
#endif /* CONFIG_LRU_GEN */
static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
-@@ -5584,7 +5886,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+@@ -5584,7 +5886,7 @@ static void shrink_lruvec(struct lruvec
bool proportional_reclaim;
struct blk_plug plug;
lru_gen_shrink_lruvec(lruvec, sc);
return;
}
-@@ -5826,6 +6128,11 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
+@@ -5826,6 +6128,11 @@ static void shrink_node(pg_data_t *pgdat
struct lruvec *target_lruvec;
bool reclaimable = false;
target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
again:
---
-2.40.0
-