mm/vmscan.c | 1010 +++++++++++++++++++++++++++++++++++-
10 files changed, 1172 insertions(+), 17 deletions(-)
-diff --git a/fs/exec.c b/fs/exec.c
-index 881390b44cfd..1afa15a07d26 100644
--- a/fs/exec.c
+++ b/fs/exec.c
-@@ -1013,6 +1013,7 @@ static int exec_mmap(struct mm_struct *mm)
+@@ -1013,6 +1013,7 @@ static int exec_mmap(struct mm_struct *m
active_mm = tsk->active_mm;
tsk->active_mm = mm;
tsk->mm = mm;
/*
* This prevents preemption while active_mm is being loaded and
* it and mm are being updated, which could cause problems for
-@@ -1028,6 +1029,7 @@ static int exec_mmap(struct mm_struct *mm)
+@@ -1028,6 +1029,7 @@ static int exec_mmap(struct mm_struct *m
tsk->mm->vmacache_seqnum = 0;
vmacache_flush(tsk);
task_unlock(tsk);
if (old_mm) {
mmap_read_unlock(old_mm);
BUG_ON(active_mm != old_mm);
-diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
-index 8d6a0329bc59..3736405cbcf6 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -348,6 +348,11 @@ struct mem_cgroup {
struct mem_cgroup_per_node *nodeinfo[];
};
-diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
-index 7f8ee09c711f..33c142d31261 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -580,6 +580,22 @@ struct mm_struct {
} __randomize_layout;
/*
-@@ -606,6 +622,66 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
+@@ -606,6 +622,66 @@ static inline cpumask_t *mm_cpumask(stru
return (struct cpumask *)&mm->cpu_bitmap;
}
struct mmu_gather;
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 4db2b877fcf9..659bab633bdf 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -385,7 +385,7 @@ enum {
ZONE_PADDING(_pad2_)
/* Per-node vmstats */
-diff --git a/include/linux/swap.h b/include/linux/swap.h
-index 4efd267e2937..e970fca4f178 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -137,6 +137,10 @@ union swap_header {
};
#ifdef __KERNEL__
-diff --git a/kernel/exit.c b/kernel/exit.c
-index 80efdfda6662..06b477395012 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
-@@ -469,6 +469,7 @@ void mm_update_next_owner(struct mm_struct *mm)
+@@ -469,6 +469,7 @@ assign_new_owner:
goto retry;
}
WRITE_ONCE(mm->owner, c);
task_unlock(c);
put_task_struct(c);
}
-diff --git a/kernel/fork.c b/kernel/fork.c
-index 68eab6ce3085..d8f37ecdde87 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
-@@ -1083,6 +1083,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
+@@ -1083,6 +1083,7 @@ static struct mm_struct *mm_init(struct
goto fail_nocontext;
mm->user_ns = get_user_ns(user_ns);
return mm;
fail_nocontext:
-@@ -1125,6 +1126,7 @@ static inline void __mmput(struct mm_struct *mm)
+@@ -1125,6 +1126,7 @@ static inline void __mmput(struct mm_str
}
if (mm->binfmt)
module_put(mm->binfmt->module);
mmdrop(mm);
}
-@@ -2622,6 +2624,13 @@ pid_t kernel_clone(struct kernel_clone_args *args)
+@@ -2622,6 +2624,13 @@ pid_t kernel_clone(struct kernel_clone_a
get_task_struct(p);
}
wake_up_new_task(p);
/* forking complete and child started to run, tell ptracer */
-diff --git a/kernel/sched/core.c b/kernel/sched/core.c
-index c1458fa8beb3..fe4d60474d4a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
-@@ -5007,6 +5007,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
+@@ -5007,6 +5007,7 @@ context_switch(struct rq *rq, struct tas
* finish_task_switch()'s mmdrop().
*/
switch_mm_irqs_off(prev->active_mm, next->mm, next);
if (!prev->mm) { // from kernel
/* will mmdrop() in finish_task_switch(). */
-diff --git a/mm/memcontrol.c b/mm/memcontrol.c
-index cc3431c5d9ba..ed87d1256f0e 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -6212,6 +6212,30 @@ static void mem_cgroup_move_task(void)
static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
{
if (value == PAGE_COUNTER_MAX)
-@@ -6555,6 +6579,7 @@ struct cgroup_subsys memory_cgrp_subsys = {
+@@ -6555,6 +6579,7 @@ struct cgroup_subsys memory_cgrp_subsys
.css_reset = mem_cgroup_css_reset,
.css_rstat_flush = mem_cgroup_css_rstat_flush,
.can_attach = mem_cgroup_can_attach,
.cancel_attach = mem_cgroup_cancel_attach,
.post_attach = mem_cgroup_move_task,
.dfl_cftypes = memory_files,
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 1d0b25ae378c..a7844c689522 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -50,6 +50,8 @@
#include <asm/tlbflush.h>
#include <asm/div64.h>
-@@ -2853,7 +2855,7 @@ static bool can_age_anon_pages(struct pglist_data *pgdat,
+@@ -2853,7 +2855,7 @@ static bool can_age_anon_pages(struct pg
for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
{
struct pglist_data *pgdat = NODE_DATA(nid);
-@@ -2898,6 +2900,371 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
- get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS;
+@@ -2899,6 +2901,371 @@ static bool __maybe_unused seq_is_valid(
}
-+/******************************************************************************
+ /******************************************************************************
+ * mm_struct list
+ ******************************************************************************/
+
+ return success;
+}
+
- /******************************************************************************
++/******************************************************************************
* refault feedback loop
******************************************************************************/
-@@ -3048,6 +3415,118 @@ static int page_inc_gen(struct lruvec *lruvec, struct page *page, bool reclaimin
+
+@@ -3048,6 +3415,118 @@ static int page_inc_gen(struct lruvec *l
return new_gen;
}
static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr)
{
unsigned long pfn = pte_pfn(pte);
-@@ -3066,8 +3545,28 @@ static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned
+@@ -3066,8 +3545,28 @@ static unsigned long get_pte_pfn(pte_t p
return pfn;
}
{
struct page *page;
-@@ -3082,9 +3581,375 @@ static struct page *get_pfn_page(unsigned long pfn, struct mem_cgroup *memcg,
+@@ -3082,9 +3581,375 @@ static struct page *get_pfn_page(unsigne
if (page_memcg_rcu(page) != memcg)
return NULL;
static void inc_min_seq(struct lruvec *lruvec, int type)
{
struct lru_gen_struct *lrugen = &lruvec->lrugen;
-@@ -3136,7 +4001,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
+@@ -3136,7 +4001,7 @@ next:
return success;
}
{
int prev, next;
int type, zone;
-@@ -3146,9 +4011,6 @@ static void inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, bool can_s
+@@ -3146,9 +4011,6 @@ static void inc_max_seq(struct lruvec *l
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
for (type = ANON_AND_FILE - 1; type >= 0; type--) {
if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
continue;
-@@ -3186,10 +4048,76 @@ static void inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, bool can_s
+@@ -3186,10 +4048,76 @@ static void inc_max_seq(struct lruvec *l
/* make sure preceding modifications appear */
smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq,
struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
{
-@@ -3265,7 +4193,7 @@ static void age_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+@@ -3265,7 +4193,7 @@ static void age_lruvec(struct lruvec *lr
need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan);
if (need_aging)
}
static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
-@@ -3274,6 +4202,8 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -3274,6 +4202,8 @@ static void lru_gen_age_node(struct pgli
VM_WARN_ON_ONCE(!current_is_kswapd());
memcg = mem_cgroup_iter(NULL, NULL, NULL);
do {
struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
-@@ -3282,11 +4212,16 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -3282,11 +4212,16 @@ static void lru_gen_age_node(struct pgli
cond_resched();
} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
*/
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
{
-@@ -3295,6 +4230,8 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -3295,6 +4230,8 @@ void lru_gen_look_around(struct page_vma
unsigned long start;
unsigned long end;
unsigned long addr;
unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
struct page *page = pvmw->page;
struct mem_cgroup *memcg = page_memcg(page);
-@@ -3309,6 +4246,9 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -3309,6 +4246,9 @@ void lru_gen_look_around(struct page_vma
if (spin_is_contended(pvmw->ptl))
return;
start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start);
end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1;
-@@ -3338,13 +4278,15 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -3338,13 +4278,15 @@ void lru_gen_look_around(struct page_vma
if (!pte_young(pte[i]))
continue;
if (pte_dirty(pte[i]) && !PageDirty(page) &&
!(PageAnon(page) && PageSwapBacked(page) &&
!PageSwapCache(page)))
-@@ -3360,7 +4302,11 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -3360,7 +4302,11 @@ void lru_gen_look_around(struct page_vma
arch_leave_lazy_mmu_mode();
rcu_read_unlock();
for_each_set_bit(i, bitmap, MIN_LRU_BATCH) {
page = pte_page(pte[i]);
activate_page(page);
-@@ -3372,8 +4318,10 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -3372,8 +4318,10 @@ void lru_gen_look_around(struct page_vma
if (!mem_cgroup_trylock_pages(memcg))
return;
for_each_set_bit(i, bitmap, MIN_LRU_BATCH) {
page = compound_head(pte_page(pte[i]));
-@@ -3384,10 +4332,14 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -3384,10 +4332,14 @@ void lru_gen_look_around(struct page_vma
if (old_gen < 0 || old_gen == new_gen)
continue;
mem_cgroup_unlock_pages();
}
-@@ -3670,6 +4622,7 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp
+@@ -3670,6 +4622,7 @@ static int evict_pages(struct lruvec *lr
struct page *page;
enum vm_event_item item;
struct reclaim_stat stat;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
-@@ -3706,6 +4659,10 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp
+@@ -3706,6 +4659,10 @@ static int evict_pages(struct lruvec *lr
move_pages_to_lru(lruvec, &list);
item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
if (!cgroup_reclaim(sc))
__count_vm_events(item, reclaimed);
-@@ -3722,6 +4679,11 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp
+@@ -3722,6 +4679,11 @@ static int evict_pages(struct lruvec *lr
return scanned;
}
static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
bool can_swap)
{
-@@ -3747,7 +4709,8 @@ static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *
+@@ -3747,7 +4709,8 @@ static unsigned long get_nr_to_scan(stru
if (current_is_kswapd())
return 0;
done:
return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0;
}
-@@ -3761,6 +4724,8 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -3761,6 +4724,8 @@ static void lru_gen_shrink_lruvec(struct
blk_start_plug(&plug);
while (true) {
int delta;
int swappiness;
-@@ -3788,6 +4753,8 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -3788,6 +4753,8 @@ static void lru_gen_shrink_lruvec(struct
cond_resched();
}
blk_finish_plug(&plug);
}
-@@ -3804,15 +4771,21 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -3804,15 +4771,21 @@ void lru_gen_init_lruvec(struct lruvec *
for_each_gen_type_zone(gen, type, zone)
INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]);
int nid;
for_each_node(nid) {
-@@ -3820,6 +4793,11 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -3820,6 +4793,11 @@ void lru_gen_exit_memcg(struct mem_cgrou
VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
sizeof(lruvec->lrugen.nr_pages)));
}
}
#endif
---
-2.40.0
-