mm/workingset.c | 4 ++--
4 files changed, 24 insertions(+), 24 deletions(-)
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index e8ed225d8f7ca..f63968bd7de59 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -178,7 +178,7 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli
+@@ -178,7 +178,7 @@ static inline void lru_gen_update_size(s
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
enum lru_list lru = type * LRU_INACTIVE_FILE;
VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS);
-@@ -224,7 +224,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
+@@ -224,7 +224,7 @@ static inline bool lru_gen_add_folio(str
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 5f74891556f33..bd3e4689f72dc 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -404,7 +404,7 @@ enum {
/* to concurrently iterate lru_gen_mm_list */
struct lru_gen_mm_state mm_state;
#endif
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index d18296109aa7e..27142caf284c1 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3190,7 +3190,7 @@ static int get_nr_gens(struct lruvec *lruvec, int type)
+@@ -3190,7 +3190,7 @@ static int get_nr_gens(struct lruvec *lr
static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
{
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
pos->refaulted = lrugen->avg_refaulted[type][tier] +
-@@ -3611,7 +3611,7 @@ static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
+@@ -3611,7 +3611,7 @@ static void read_ctrl_pos(struct lruvec
static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
{
int hist, tier;
bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1;
unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
-@@ -3688,7 +3688,7 @@ static int folio_update_gen(struct folio *folio, int gen)
+@@ -3688,7 +3688,7 @@ static int folio_update_gen(struct folio
static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
int type = folio_is_file_lru(folio);
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
-@@ -3733,7 +3733,7 @@ static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
+@@ -3733,7 +3733,7 @@ static void update_batch_size(struct lru
static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk)
{
int gen, type, zone;
walk->batched = 0;
-@@ -4250,7 +4250,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
+@@ -4250,7 +4250,7 @@ static bool inc_min_seq(struct lruvec *l
{
int zone;
int remaining = MAX_LRU_BATCH;
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
if (type == LRU_GEN_ANON && !can_swap)
-@@ -4286,7 +4286,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
+@@ -4286,7 +4286,7 @@ static bool try_to_inc_min_seq(struct lr
{
int gen, type, zone;
bool success = false;
DEFINE_MIN_SEQ(lruvec);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
-@@ -4307,7 +4307,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
+@@ -4307,7 +4307,7 @@ next:
;
}
if (can_swap) {
min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]);
min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
-@@ -4329,7 +4329,7 @@ static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)
+@@ -4329,7 +4329,7 @@ static void inc_max_seq(struct lruvec *l
{
int prev, next;
int type, zone;
spin_lock_irq(&lruvec->lru_lock);
-@@ -4387,7 +4387,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4387,7 +4387,7 @@ static bool try_to_inc_max_seq(struct lr
bool success;
struct lru_gen_mm_walk *walk;
struct mm_struct *mm = NULL;
VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
-@@ -4452,7 +4452,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig
+@@ -4452,7 +4452,7 @@ static bool should_run_aging(struct lruv
unsigned long old = 0;
unsigned long young = 0;
unsigned long total = 0;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
for (type = !can_swap; type < ANON_AND_FILE; type++) {
-@@ -4737,7 +4737,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+@@ -4737,7 +4737,7 @@ static bool sort_folio(struct lruvec *lr
int delta = folio_nr_pages(folio);
int refs = folio_lru_refs(folio);
int tier = lru_tier_from_refs(refs);
VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
-@@ -4837,7 +4837,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
+@@ -4837,7 +4837,7 @@ static int scan_folios(struct lruvec *lr
int scanned = 0;
int isolated = 0;
int remaining = MAX_LRU_BATCH;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
VM_WARN_ON_ONCE(!list_empty(list));
-@@ -5237,7 +5237,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5237,7 +5237,7 @@ done:
static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
{
if (lrugen->enabled) {
enum lru_list lru;
-@@ -5519,7 +5519,7 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
+@@ -5519,7 +5519,7 @@ static void lru_gen_seq_show_full(struct
int i;
int type, tier;
int hist = lru_hist_from_seq(seq);
for (tier = 0; tier < MAX_NR_TIERS; tier++) {
seq_printf(m, " %10d", tier);
-@@ -5569,7 +5569,7 @@ static int lru_gen_seq_show(struct seq_file *m, void *v)
+@@ -5569,7 +5569,7 @@ static int lru_gen_seq_show(struct seq_f
unsigned long seq;
bool full = !debugfs_real_fops(m->file)->write;
struct lruvec *lruvec = v;
int nid = lruvec_pgdat(lruvec)->node_id;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MAX_SEQ(lruvec);
-@@ -5823,7 +5823,7 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -5823,7 +5823,7 @@ void lru_gen_init_lruvec(struct lruvec *
{
int i;
int gen, type, zone;
lrugen->max_seq = MIN_NR_GENS + 1;
lrugen->enabled = lru_gen_enabled();
-diff --git a/mm/workingset.c b/mm/workingset.c
-index ae7e984b23c6b..688aaa73f64e8 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
-@@ -223,7 +223,7 @@ static void *lru_gen_eviction(struct folio *folio)
+@@ -223,7 +223,7 @@ static void *lru_gen_eviction(struct fol
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
int type = folio_is_file_lru(folio);
int delta = folio_nr_pages(folio);
int refs = folio_lru_refs(folio);
-@@ -252,7 +252,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
+@@ -252,7 +252,7 @@ static void lru_gen_refault(struct folio
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
struct mem_cgroup *memcg;
struct pglist_data *pgdat;
int type = folio_is_file_lru(folio);
---
-2.40.1
-
mm/vmscan.c | 20 ++++++++++----------
4 files changed, 20 insertions(+), 20 deletions(-)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index d7062c6a89464..d8f721f98868a 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
@@ -89,15 +89,15 @@ variables are monotonically increasing.
contrast to moving across generations, which requires the LRU lock,
moving across tiers only involves atomic operations on
``folio->flags`` and therefore has a negligible cost. A feedback loop
-@@ -127,7 +127,7 @@ page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``.
+@@ -127,7 +127,7 @@ page mapped by this PTE to ``(max_seq%MA
Eviction
--------
The eviction consumes old generations. Given an ``lruvec``, it
``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to
evict from, it first compares ``min_seq[]`` to select the older type.
If both types are equally old, it selects the one whose first tier has
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index f63968bd7de59..da38e3d962e2f 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -256,9 +256,9 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
+@@ -256,9 +256,9 @@ static inline bool lru_gen_add_folio(str
lru_gen_update_size(lruvec, folio, -1, gen);
/* for folio_rotate_reclaimable() */
if (reclaiming)
return true;
}
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index bd3e4689f72dc..02e4323744715 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -312,7 +312,7 @@ enum lruvec_flags {
/* the multi-gen LRU sizes, eventually consistent */
long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 27142caf284c1..b02fed912f742 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4258,7 +4258,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
+@@ -4258,7 +4258,7 @@ static bool inc_min_seq(struct lruvec *l
/* prevent cold/hot inversion if force_scan is true */
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head);
-@@ -4269,7 +4269,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
+@@ -4269,7 +4269,7 @@ static bool inc_min_seq(struct lruvec *l
VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
new_gen = folio_inc_gen(lruvec, folio, false);
if (!--remaining)
return false;
-@@ -4297,7 +4297,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
+@@ -4297,7 +4297,7 @@ static bool try_to_inc_min_seq(struct lr
gen = lru_gen_from_seq(min_seq[type]);
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
goto next;
}
-@@ -4762,7 +4762,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+@@ -4762,7 +4762,7 @@ static bool sort_folio(struct lruvec *lr
/* promoted */
if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
return true;
}
-@@ -4771,7 +4771,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+@@ -4771,7 +4771,7 @@ static bool sort_folio(struct lruvec *lr
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
gen = folio_inc_gen(lruvec, folio, false);
WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
lrugen->protected[hist][type][tier - 1] + delta);
-@@ -4783,7 +4783,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+@@ -4783,7 +4783,7 @@ static bool sort_folio(struct lruvec *lr
if (folio_test_locked(folio) || folio_test_writeback(folio) ||
(type == LRU_GEN_FILE && folio_test_dirty(folio))) {
gen = folio_inc_gen(lruvec, folio, true);
return true;
}
-@@ -4850,7 +4850,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
+@@ -4850,7 +4850,7 @@ static int scan_folios(struct lruvec *lr
for (zone = sc->reclaim_idx; zone >= 0; zone--) {
LIST_HEAD(moved);
int skipped = 0;
while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head);
-@@ -5250,7 +5250,7 @@ static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
+@@ -5250,7 +5250,7 @@ static bool __maybe_unused state_is_vali
int gen, type, zone;
for_each_gen_type_zone(gen, type, zone) {
return false;
}
}
-@@ -5295,7 +5295,7 @@ static bool drain_evictable(struct lruvec *lruvec)
+@@ -5295,7 +5295,7 @@ static bool drain_evictable(struct lruve
int remaining = MAX_LRU_BATCH;
for_each_gen_type_zone(gen, type, zone) {
while (!list_empty(head)) {
bool success;
-@@ -5832,7 +5832,7 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -5832,7 +5832,7 @@ void lru_gen_init_lruvec(struct lruvec *
lrugen->timestamps[i] = jiffies;
for_each_gen_type_zone(gen, type, zone)
lruvec->mm_state.seq = MIN_NR_GENS;
init_waitqueue_head(&lruvec->mm_state.wait);
---
-2.40.1
-
mm/vmscan.c | 81 +++++++++++++++--------------------------------------
1 file changed, 23 insertions(+), 58 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index b02fed912f742..991961180b320 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -448,6 +448,11 @@ static bool cgroup_reclaim(struct scan_control *sc)
+@@ -448,6 +448,11 @@ static bool cgroup_reclaim(struct scan_c
return sc->target_mem_cgroup;
}
/**
* writeback_throttling_sane - is the usual dirty throttling mechanism available?
* @sc: scan_control in question
-@@ -498,6 +503,11 @@ static bool cgroup_reclaim(struct scan_control *sc)
+@@ -498,6 +503,11 @@ static bool cgroup_reclaim(struct scan_c
return false;
}
static bool writeback_throttling_sane(struct scan_control *sc)
{
return true;
-@@ -4993,8 +5003,7 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int sw
+@@ -4993,8 +5003,7 @@ static int isolate_folios(struct lruvec
return scanned;
}
{
int type;
int scanned;
-@@ -5083,9 +5092,6 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
+@@ -5083,9 +5092,6 @@ retry:
goto retry;
}
return scanned;
}
-@@ -5124,67 +5130,26 @@ static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *
+@@ -5124,67 +5130,26 @@ done:
return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0;
}
lru_add_drain();
-@@ -5208,7 +5173,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5208,7 +5173,7 @@ static void lru_gen_shrink_lruvec(struct
if (!nr_to_scan)
goto done;
if (!delta)
goto done;
-@@ -5216,7 +5181,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5216,7 +5181,7 @@ static void lru_gen_shrink_lruvec(struct
if (scanned >= nr_to_scan)
break;
break;
cond_resched();
-@@ -5666,7 +5631,7 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
+@@ -5666,7 +5631,7 @@ static int run_eviction(struct lruvec *l
if (sc->nr_reclaimed >= nr_to_reclaim)
return 0;
return 0;
cond_resched();
---
-2.40.1
-
mm/vmscan.c | 126 ++++++++++++++++++++++++----------------------------
1 file changed, 59 insertions(+), 67 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 991961180b320..5a2e83e673232 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -136,7 +136,6 @@ struct scan_control {
unsigned long last_reclaimed;
#endif
-@@ -4455,7 +4454,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4455,7 +4454,7 @@ done:
return true;
}
struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
{
int gen, type, zone;
-@@ -4464,6 +4463,13 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig
+@@ -4464,6 +4463,13 @@ static bool should_run_aging(struct lruv
unsigned long total = 0;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
for (type = !can_swap; type < ANON_AND_FILE; type++) {
unsigned long seq;
-@@ -4492,8 +4498,6 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig
+@@ -4492,8 +4498,6 @@ static bool should_run_aging(struct lruv
* stalls when the number of generations reaches MIN_NR_GENS. Hence, the
* ideal number of generations is MIN_NR_GENS+1.
*/
if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
return false;
-@@ -4512,40 +4516,54 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig
+@@ -4512,40 +4516,54 @@ static bool should_run_aging(struct lruv
return false;
}
}
/* to protect the working set of the last N jiffies */
-@@ -4554,46 +4572,32 @@ static unsigned long lru_gen_min_ttl __read_mostly;
+@@ -4554,46 +4572,32 @@ static unsigned long lru_gen_min_ttl __r
static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
{
struct mem_cgroup *memcg;
*/
if (mutex_trylock(&oom_lock)) {
struct oom_control oc = {
-@@ -5101,33 +5105,27 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
+@@ -5101,33 +5105,27 @@ retry:
* reclaim.
*/
static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
}
static unsigned long get_nr_to_reclaim(struct scan_control *sc)
-@@ -5146,9 +5144,7 @@ static unsigned long get_nr_to_reclaim(struct scan_control *sc)
+@@ -5146,9 +5144,7 @@ static unsigned long get_nr_to_reclaim(s
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
{
struct blk_plug plug;
unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
lru_add_drain();
-@@ -5169,13 +5165,13 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5169,13 +5165,13 @@ static void lru_gen_shrink_lruvec(struct
else
swappiness = 0;
scanned += delta;
if (scanned >= nr_to_scan)
-@@ -5187,10 +5183,6 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5187,10 +5183,6 @@ static void lru_gen_shrink_lruvec(struct
cond_resched();
}
clear_mm_walk();
blk_finish_plug(&plug);
---
-2.40.1
-
mm/vmscan.c | 124 ++++++++++++++++++++++++++--------------------------
1 file changed, 62 insertions(+), 62 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 5a2e83e673232..0c47952714b26 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4454,68 +4454,6 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4454,68 +4454,6 @@ done:
return true;
}
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
{
int gen, type, zone;
-@@ -5099,6 +5037,68 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
+@@ -5099,6 +5037,68 @@ retry:
return scanned;
}
/*
* For future optimizations:
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
---
-2.40.1
-
mm/vmscan.c | 374 +++++++++++++++++++++++++++++++++----
6 files changed, 500 insertions(+), 35 deletions(-)
-diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
-index e039763029563..82d28b052a9e5 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
-@@ -790,6 +790,11 @@ static inline void obj_cgroup_put(struct obj_cgroup *objcg)
+@@ -790,6 +790,11 @@ static inline void obj_cgroup_put(struct
percpu_ref_put(&objcg->refcnt);
}
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
if (memcg)
-@@ -1290,6 +1295,11 @@ static inline void obj_cgroup_put(struct obj_cgroup *objcg)
+@@ -1290,6 +1295,11 @@ static inline void obj_cgroup_put(struct
{
}
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index da38e3d962e2f..c1fd3922dc5dd 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -122,6 +122,18 @@ static inline bool lru_gen_in_fault(void)
+@@ -122,6 +122,18 @@ static inline bool lru_gen_in_fault(void
return current->in_lru_fault;
}
static inline int lru_gen_from_seq(unsigned long seq)
{
return seq % MAX_NR_GENS;
-@@ -297,6 +309,11 @@ static inline bool lru_gen_in_fault(void)
+@@ -297,6 +309,11 @@ static inline bool lru_gen_in_fault(void
return false;
}
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 02e4323744715..66e067a635682 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -7,6 +7,7 @@
};
enum {
-@@ -479,12 +497,87 @@ void lru_gen_init_lruvec(struct lruvec *lruvec);
+@@ -479,12 +497,87 @@ void lru_gen_init_lruvec(struct lruvec *
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
#ifdef CONFIG_MEMCG
static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
{
}
-@@ -494,6 +587,7 @@ static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -494,6 +587,7 @@ static inline void lru_gen_look_around(s
}
#ifdef CONFIG_MEMCG
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
}
-@@ -501,7 +595,24 @@ static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
+@@ -501,7 +595,24 @@ static inline void lru_gen_init_memcg(st
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
}
#endif
CACHELINE_PADDING(_pad2_);
-diff --git a/mm/memcontrol.c b/mm/memcontrol.c
-index 3e8f1ad0fe9db..7815d556e38cc 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
-@@ -477,6 +477,16 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
+@@ -477,6 +477,16 @@ static void mem_cgroup_update_tree(struc
struct mem_cgroup_per_node *mz;
struct mem_cgroup_tree_per_node *mctz;
mctz = soft_limit_tree.rb_tree_per_node[nid];
if (!mctz)
return;
-@@ -3522,6 +3532,9 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
+@@ -3522,6 +3532,9 @@ unsigned long mem_cgroup_soft_limit_recl
struct mem_cgroup_tree_per_node *mctz;
unsigned long excess;
if (order > 0)
return 0;
-@@ -5382,6 +5395,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
+@@ -5382,6 +5395,7 @@ static int mem_cgroup_css_online(struct
if (unlikely(mem_cgroup_is_root(memcg)))
queue_delayed_work(system_unbound_wq, &stats_flush_dwork,
2UL*HZ);
return 0;
offline_kmem:
memcg_offline_kmem(memcg);
-@@ -5413,6 +5427,7 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
+@@ -5413,6 +5427,7 @@ static void mem_cgroup_css_offline(struc
memcg_offline_kmem(memcg);
reparent_shrinker_deferred(memcg);
wb_memcg_offline(memcg);
drain_all_stock(memcg);
-@@ -5424,6 +5439,7 @@ static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
+@@ -5424,6 +5439,7 @@ static void mem_cgroup_css_released(stru
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
invalidate_reclaim_iterators(memcg);
}
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
-diff --git a/mm/page_alloc.c b/mm/page_alloc.c
-index 69668817fed37..473057b81a9df 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
-@@ -7957,6 +7957,7 @@ static void __init free_area_init_node(int nid)
+@@ -7957,6 +7957,7 @@ static void __init free_area_init_node(i
pgdat_set_deferred_range(pgdat);
free_area_init_core(pgdat);
}
static void __init free_area_init_memoryless_node(int nid)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 0c47952714b26..65eb28448f216 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -54,6 +54,8 @@
/* Allocation order */
s8 order;
-@@ -3160,6 +3157,9 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS);
+@@ -3160,6 +3157,9 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_ca
for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
{
struct pglist_data *pgdat = NODE_DATA(nid);
-@@ -4440,8 +4440,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4440,8 +4440,7 @@ done:
if (sc->priority <= DEF_PRIORITY - 2)
wait_event_killable(lruvec->mm_state.wait,
max_seq < READ_ONCE(lrugen->max_seq));
}
VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq));
-@@ -4514,8 +4513,6 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -4514,8 +4513,6 @@ static void lru_gen_age_node(struct pgli
VM_WARN_ON_ONCE(!current_is_kswapd());
/* check the order to exclude compaction-induced reclaim */
if (!min_ttl || sc->order || sc->priority == DEF_PRIORITY)
return;
-@@ -5104,8 +5101,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
+@@ -5104,8 +5101,7 @@ static bool should_run_aging(struct lruv
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
* reclaim.
*/
{
unsigned long nr_to_scan;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
-@@ -5122,10 +5118,8 @@ static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *
+@@ -5122,10 +5118,8 @@ static unsigned long get_nr_to_scan(stru
if (sc->priority == DEF_PRIORITY)
return nr_to_scan;
}
static unsigned long get_nr_to_reclaim(struct scan_control *sc)
-@@ -5134,29 +5128,18 @@ static unsigned long get_nr_to_reclaim(struct scan_control *sc)
+@@ -5134,29 +5128,18 @@ static unsigned long get_nr_to_reclaim(s
if (!global_reclaim(sc))
return -1;
if (sc->may_swap)
swappiness = get_swappiness(lruvec, sc);
-@@ -5166,7 +5149,7 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5166,7 +5149,7 @@ static void lru_gen_shrink_lruvec(struct
swappiness = 0;
nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
break;
delta = evict_folios(lruvec, sc, swappiness);
-@@ -5183,11 +5166,252 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5183,10 +5166,251 @@ static void lru_gen_shrink_lruvec(struct
cond_resched();
}
+ if (try_to_shrink_lruvec(lruvec, sc))
+ lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
+
- clear_mm_walk();
-
- blk_finish_plug(&plug);
- }
-
++ clear_mm_walk();
++
++ blk_finish_plug(&plug);
++}
++
+#else /* !CONFIG_MEMCG */
+
+static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
+ if (current_is_kswapd())
+ sc->nr_reclaimed += reclaimed;
+
-+ clear_mm_walk();
-+
-+ blk_finish_plug(&plug);
+ clear_mm_walk();
+
+ blk_finish_plug(&plug);
+
+ /* kswapd should never fail */
+ pgdat->kswapd_failures = 0;
+ WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
+
+ spin_unlock(&pgdat->memcg_lru.lock);
-+}
+ }
+#endif
-+
+
/******************************************************************************
* state change
- ******************************************************************************/
-@@ -5644,11 +5868,11 @@ static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq,
+@@ -5644,11 +5868,11 @@ static int run_cmd(char cmd, int memcg_i
if (!mem_cgroup_disabled()) {
rcu_read_lock();
rcu_read_unlock();
if (!memcg)
-@@ -5796,6 +6020,19 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -5796,6 +6020,19 @@ void lru_gen_init_lruvec(struct lruvec *
}
#ifdef CONFIG_MEMCG
void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
INIT_LIST_HEAD(&memcg->mm_list.fifo);
-@@ -5819,7 +6056,69 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -5819,7 +6056,69 @@ void lru_gen_exit_memcg(struct mem_cgrou
}
}
}
static int __init init_lru_gen(void)
{
-@@ -5846,6 +6145,10 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5846,6 +6145,10 @@ static void lru_gen_shrink_lruvec(struct
{
}
#endif /* CONFIG_LRU_GEN */
static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
-@@ -5859,7 +6162,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+@@ -5859,7 +6162,7 @@ static void shrink_lruvec(struct lruvec
bool proportional_reclaim;
struct blk_plug plug;
lru_gen_shrink_lruvec(lruvec, sc);
return;
}
-@@ -6102,6 +6405,11 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
+@@ -6102,6 +6405,11 @@ static void shrink_node(pg_data_t *pgdat
struct lruvec *target_lruvec;
bool reclaimable = false;
target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
again:
---
-2.40.1
-
mm/vmscan.c | 55 +++++++++++++++++++++++++++--------------------------
1 file changed, 28 insertions(+), 27 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 65eb28448f216..0a0e1250ffc87 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3185,6 +3185,9 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
+@@ -3185,6 +3185,9 @@ static int get_swappiness(struct lruvec
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
if (!can_demote(pgdat->node_id, sc) &&
mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
return 0;
-@@ -4223,7 +4226,7 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_
+@@ -4223,7 +4226,7 @@ static void walk_mm(struct lruvec *lruve
} while (err == -EAGAIN);
}
{
struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
-@@ -4231,7 +4234,7 @@ static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat)
+@@ -4231,7 +4234,7 @@ static struct lru_gen_mm_walk *set_mm_wa
VM_WARN_ON_ONCE(walk);
walk = &pgdat->mm_walk;
VM_WARN_ON_ONCE(current_is_kswapd());
walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
-@@ -4417,7 +4420,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4417,7 +4420,7 @@ static bool try_to_inc_max_seq(struct lr
goto done;
}
if (!walk) {
success = iterate_mm_list_nowalk(lruvec, max_seq);
goto done;
-@@ -4486,8 +4489,6 @@ static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc
+@@ -4486,8 +4489,6 @@ static bool lruvec_is_reclaimable(struct
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MIN_SEQ(lruvec);
/* see the comment on lru_gen_folio */
gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]);
birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
-@@ -4743,12 +4744,8 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
+@@ -4743,12 +4744,8 @@ static bool isolate_folio(struct lruvec
{
bool success;
(folio_test_dirty(folio) ||
(folio_test_anon(folio) && !folio_test_swapcache(folio))))
return false;
-@@ -4845,9 +4842,8 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
+@@ -4845,9 +4842,8 @@ static int scan_folios(struct lruvec *lr
__count_vm_events(PGSCAN_ANON + type, isolated);
/*
*/
return isolated || !remaining ? scanned : 0;
}
-@@ -5107,8 +5103,7 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool
+@@ -5107,8 +5103,7 @@ static long get_nr_to_scan(struct lruvec
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MAX_SEQ(lruvec);
return 0;
if (!should_run_aging(lruvec, max_seq, sc, can_swap, &nr_to_scan))
-@@ -5136,17 +5131,14 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+@@ -5136,17 +5131,14 @@ static bool try_to_shrink_lruvec(struct
long nr_to_scan;
unsigned long scanned = 0;
unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
if (nr_to_scan <= 0)
-@@ -5277,12 +5269,13 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc
+@@ -5277,12 +5269,13 @@ static void lru_gen_shrink_lruvec(struct
struct blk_plug plug;
VM_WARN_ON_ONCE(global_reclaim(sc));
if (try_to_shrink_lruvec(lruvec, sc))
lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
-@@ -5338,11 +5331,19 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
+@@ -5338,11 +5331,19 @@ static void lru_gen_shrink_node(struct p
VM_WARN_ON_ONCE(!global_reclaim(sc));
set_initial_priority(pgdat, sc);
-@@ -5360,7 +5361,7 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
+@@ -5360,7 +5361,7 @@ static void lru_gen_shrink_node(struct p
clear_mm_walk();
blk_finish_plug(&plug);
/* kswapd should never fail */
pgdat->kswapd_failures = 0;
}
-@@ -5932,7 +5933,7 @@ static ssize_t lru_gen_seq_write(struct file *file, const char __user *src,
+@@ -5932,7 +5933,7 @@ static ssize_t lru_gen_seq_write(struct
set_task_reclaim_state(current, &sc.reclaim_state);
flags = memalloc_noreclaim_save();
blk_start_plug(&plug);
err = -ENOMEM;
goto done;
}
---
-2.40.1
-
mm/vmscan.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 0a0e1250ffc87..aa9746f2bc80b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4415,7 +4415,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4415,7 +4415,7 @@ static bool try_to_inc_max_seq(struct lr
* handful of PTEs. Spreading the work out over a period of time usually
* is less efficient, but it avoids bursty page faults.
*/
success = iterate_mm_list_nowalk(lruvec, max_seq);
goto done;
}
---
-2.40.1
-
mm/vmscan.c | 25 +++++++++++++++----------
1 file changed, 15 insertions(+), 10 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index aa9746f2bc80b..49da02f841c81 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -5206,18 +5206,20 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
+@@ -5206,18 +5206,20 @@ static int shrink_one(struct lruvec *lru
static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
{
gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq));
rcu_read_lock();
-@@ -5241,14 +5243,22 @@ static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -5241,14 +5243,22 @@ restart:
op = shrink_one(lruvec, sc);
/* restart if raced with lru_gen_rotate_memcg() */
if (gen != get_nulls_value(pos))
goto restart;
-@@ -5257,11 +5267,6 @@ static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -5257,11 +5267,6 @@ restart:
bin = get_memcg_bin(bin + 1);
if (bin != first_bin)
goto restart;
}
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
---
-2.40.1
-
mm/vmscan.c | 5 ++++-
4 files changed, 33 insertions(+), 29 deletions(-)
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index c1fd3922dc5dd..7bb2e5f94734c 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -595,4 +595,12 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
+@@ -595,4 +595,12 @@ pte_install_uffd_wp_if_needed(struct vm_
#endif
}
+}
+
#endif
-diff --git a/mm/memory.c b/mm/memory.c
-index 747b7ea30f890..c2f48f8003c2e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
-@@ -1435,8 +1435,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
+@@ -1435,8 +1435,7 @@ again:
force_flush = 1;
set_page_dirty(page);
}
mark_page_accessed(page);
}
rss[mm_counter(page)]--;
-@@ -5170,8 +5169,8 @@ static inline void mm_account_fault(struct pt_regs *regs,
+@@ -5170,8 +5169,8 @@ static inline void mm_account_fault(stru
#ifdef CONFIG_LRU_GEN
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
}
static void lru_gen_exit_fault(void)
-diff --git a/mm/rmap.c b/mm/rmap.c
-index 7da2d8d097d9b..825dac3caa1e5 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
-@@ -823,25 +823,14 @@ static bool folio_referenced_one(struct folio *folio,
+@@ -823,25 +823,14 @@ static bool folio_referenced_one(struct
}
if (pvmw.pte) {
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address,
pvmw.pmd))
-@@ -875,7 +864,20 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
+@@ -875,7 +864,20 @@ static bool invalid_folio_referenced_vma
struct folio_referenced_arg *pra = arg;
struct mem_cgroup *memcg = pra->memcg;
return true;
return false;
-@@ -906,6 +908,7 @@ int folio_referenced(struct folio *folio, int is_locked,
+@@ -906,6 +908,7 @@ int folio_referenced(struct folio *folio
.arg = (void *)&pra,
.anon_lock = folio_lock_anon_vma_read,
.try_lock = true,
};
*vm_flags = 0;
-@@ -921,15 +924,6 @@ int folio_referenced(struct folio *folio, int is_locked,
+@@ -921,15 +924,6 @@ int folio_referenced(struct folio *folio
return 1;
}
rmap_walk(folio, &rwc);
*vm_flags = pra.vm_flags;
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 49da02f841c81..596fed6ae0439 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3778,7 +3778,10 @@ static int should_skip_vma(unsigned long start, unsigned long end, struct mm_wal
+@@ -3778,7 +3778,10 @@ static int should_skip_vma(unsigned long
if (is_vm_hugetlb_page(vma))
return true;
return true;
if (vma == get_gate_vma(vma->vm_mm))
---
-2.40.1
-
mm/fadvise.c | 5 ++++-
3 files changed, 9 insertions(+), 1 deletion(-)
-diff --git a/include/linux/fs.h b/include/linux/fs.h
-index f14ecbeab2a9d..97f9c41c1a43a 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
-@@ -166,6 +166,8 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
+@@ -166,6 +166,8 @@ typedef int (dio_iodone_t)(struct kiocb
/* File supports DIRECT IO */
#define FMODE_CAN_ODIRECT ((__force fmode_t)0x400000)
/* File was opened by fanotify and shouldn't generate fanotify events */
#define FMODE_NONOTIFY ((__force fmode_t)0x4000000)
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index 7bb2e5f94734c..9a8e2049333c0 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -600,6 +600,9 @@ static inline bool vma_has_recency(struct vm_area_struct *vma)
+@@ -600,6 +600,9 @@ static inline bool vma_has_recency(struc
if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))
return false;
return true;
}
-diff --git a/mm/fadvise.c b/mm/fadvise.c
-index c76ee665355a4..2ba24d865bf5f 100644
--- a/mm/fadvise.c
+++ b/mm/fadvise.c
-@@ -80,7 +80,7 @@ int generic_fadvise(struct file *file, loff_t offset, loff_t len, int advice)
+@@ -80,7 +80,7 @@ int generic_fadvise(struct file *file, l
case POSIX_FADV_NORMAL:
file->f_ra.ra_pages = bdi->ra_pages;
spin_lock(&file->f_lock);
spin_unlock(&file->f_lock);
break;
case POSIX_FADV_RANDOM:
-@@ -107,6 +107,9 @@ int generic_fadvise(struct file *file, loff_t offset, loff_t len, int advice)
+@@ -107,6 +107,9 @@ int generic_fadvise(struct file *file, l
force_page_cache_readahead(mapping, file, start_index, nrpages);
break;
case POSIX_FADV_NOREUSE:
break;
case POSIX_FADV_DONTNEED:
__filemap_fdatawrite_range(mapping, offset, endbyte,
---
-2.40.1
-
mm/vmscan.c | 4 ++++
2 files changed, 19 insertions(+)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index d8f721f98868a..6e1483e70fdca 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
-@@ -141,6 +141,21 @@ loop has detected outlying refaults from the tier this page is in. To
+@@ -141,6 +141,21 @@ loop has detected outlying refaults from
this end, the feedback loop uses the first tier as the baseline, for
the reason stated earlier.
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 596fed6ae0439..ab0b8d3b9d88f 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4459,6 +4459,10 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4459,6 +4459,10 @@ done:
return true;
}
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
{
int gen, type, zone;
---
-2.40.1
-
mm/vmscan.c | 4 ++++
2 files changed, 18 insertions(+)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index 6e1483e70fdca..bd988a142bc2f 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
-@@ -156,6 +156,20 @@ This time-based approach has the following advantages:
+@@ -156,6 +156,20 @@ This time-based approach has the followi
and memory sizes.
2. It is more reliable because it is directly wired to the OOM killer.
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index ab0b8d3b9d88f..8fa82630240d6 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4553,6 +4553,10 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -4553,6 +4553,10 @@ static void lru_gen_age_node(struct pgli
}
}
/*
* This function exploits spatial locality when shrink_folio_list() walks the
* rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If
---
-2.40.1
-
mm/vmscan.c | 180 +++++++++++++++---------------
2 files changed, 108 insertions(+), 88 deletions(-)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index bd988a142bc2f..770b5d539856c 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
-@@ -170,6 +170,22 @@ promotes hot pages. If the scan was done cacheline efficiently, it
+@@ -170,6 +170,22 @@ promotes hot pages. If the scan was done
adds the PMD entry pointing to the PTE table to the Bloom filter. This
forms a feedback loop between the eviction and the aging.
Summary
-------
The multi-gen LRU can be disassembled into the following parts:
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 8fa82630240d6..74b4f9d660b56 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3208,6 +3208,98 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
- get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS;
+@@ -3209,6 +3209,98 @@ static bool __maybe_unused seq_is_valid(
}
-+/******************************************************************************
+ /******************************************************************************
+ * Bloom filters
+ ******************************************************************************/
+
+ WRITE_ONCE(lruvec->mm_state.filters[gen], filter);
+}
+
- /******************************************************************************
++/******************************************************************************
* mm_struct list
******************************************************************************/
-@@ -3333,94 +3425,6 @@ void lru_gen_migrate_mm(struct mm_struct *mm)
+
+@@ -3333,94 +3425,6 @@ void lru_gen_migrate_mm(struct mm_struct
}
#endif
static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last)
{
int i;
---
-2.40.1
-
mm/vmscan.c | 250 +++++++++++++++++-------------
5 files changed, 178 insertions(+), 143 deletions(-)
-diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
-index 770b5d539856c..5f1f6ecbb79b9 100644
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
-@@ -186,9 +186,40 @@ is false positive, the cost is an additional scan of a range of PTEs,
+@@ -186,9 +186,40 @@ is false positive, the cost is an additi
which may yield hot pages anyway. Parameters of the filter itself can
control the false positive rate in the limit.
* Generations
* Rmap walks
-diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
-index 9a8e2049333c0..5567f4850243b 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
-@@ -122,18 +122,6 @@ static inline bool lru_gen_in_fault(void)
+@@ -122,18 +122,6 @@ static inline bool lru_gen_in_fault(void
return current->in_lru_fault;
}
static inline int lru_gen_from_seq(unsigned long seq)
{
return seq % MAX_NR_GENS;
-@@ -309,11 +297,6 @@ static inline bool lru_gen_in_fault(void)
+@@ -309,11 +297,6 @@ static inline bool lru_gen_in_fault(void
return false;
}
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 66e067a635682..403c7461e7a70 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -368,15 +368,6 @@ struct page_vma_mapped_walk;
#ifdef CONFIG_LRU_GEN
enum {
-@@ -557,7 +548,7 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg);
+@@ -557,7 +548,7 @@ void lru_gen_exit_memcg(struct mem_cgrou
void lru_gen_online_memcg(struct mem_cgroup *memcg);
void lru_gen_offline_memcg(struct mem_cgroup *memcg);
void lru_gen_release_memcg(struct mem_cgroup *memcg);
#else /* !CONFIG_MEMCG */
-@@ -608,7 +599,7 @@ static inline void lru_gen_release_memcg(struct mem_cgroup *memcg)
+@@ -608,7 +599,7 @@ static inline void lru_gen_release_memcg
{
}
{
}
-diff --git a/mm/memcontrol.c b/mm/memcontrol.c
-index 7815d556e38cc..5397aeb43986d 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
-@@ -478,12 +478,8 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
+@@ -478,12 +478,8 @@ static void mem_cgroup_update_tree(struc
struct mem_cgroup_tree_per_node *mctz;
if (lru_gen_enabled()) {
return;
}
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 74b4f9d660b56..ccde215c084ca 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4689,6 +4689,148 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
- mem_cgroup_unlock_pages();
+@@ -4690,6 +4690,148 @@ void lru_gen_look_around(struct page_vma
}
-+/******************************************************************************
+ /******************************************************************************
+ * memcg LRU
+ ******************************************************************************/
+
+
+#endif
+
- /******************************************************************************
++/******************************************************************************
* the eviction
******************************************************************************/
-@@ -5386,53 +5528,6 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
+
+@@ -5386,53 +5528,6 @@ done:
pgdat->kswapd_failures = 0;
}
/******************************************************************************
* state change
******************************************************************************/
-@@ -6078,67 +6173,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -6078,67 +6173,6 @@ void lru_gen_exit_memcg(struct mem_cgrou
}
}
#endif /* CONFIG_MEMCG */
static int __init init_lru_gen(void)
---
-2.40.1
-
mm/vmscan.c | 5 +++++
1 file changed, 5 insertions(+)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index ccde215c084ca..d5d6f8d94f58a 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -6160,12 +6160,17 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -6160,12 +6160,17 @@ void lru_gen_exit_memcg(struct mem_cgrou
int i;
int nid;
for (i = 0; i < NR_BLOOM_FILTERS; i++) {
bitmap_free(lruvec->mm_state.filters[i]);
lruvec->mm_state.filters[i] = NULL;
---
-2.40.1
-
mm/vmscan.c | 40 ++++++++++++++++++++--------------------
1 file changed, 20 insertions(+), 20 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index d5d6f8d94f58a..8f496c2e670a9 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3980,8 +3980,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
+@@ -3980,8 +3980,8 @@ restart:
}
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
{
int i;
pmd_t *pmd;
-@@ -3994,18 +3994,19 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
+@@ -3994,18 +3994,19 @@ static void walk_pmd_range_locked(pud_t
VM_WARN_ON_ONCE(pud_leaf(*pud));
/* try to batch at most 1+MIN_LRU_BATCH+1 entries */
ptl = pmd_lockptr(args->mm, pmd);
if (!spin_trylock(ptl))
-@@ -4016,15 +4017,16 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
+@@ -4016,15 +4017,16 @@ static void walk_pmd_range_locked(pud_t
do {
unsigned long pfn;
struct folio *folio;
pmdp_test_and_clear_young(vma, addr, pmd + i);
goto next;
}
-@@ -4053,12 +4055,11 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
+@@ -4053,12 +4055,11 @@ next:
arch_leave_lazy_mmu_mode();
spin_unlock(ptl);
done:
{
}
#endif
-@@ -4071,9 +4072,9 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+@@ -4071,9 +4072,9 @@ static void walk_pmd_range(pud_t *pud, u
unsigned long next;
unsigned long addr;
struct vm_area_struct *vma;
VM_WARN_ON_ONCE(pud_leaf(*pud));
-@@ -4115,18 +4116,17 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+@@ -4115,18 +4116,17 @@ restart:
if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
continue;
}
if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
-@@ -4143,7 +4143,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+@@ -4143,7 +4143,7 @@ restart:
update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
}
if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
goto restart;
---
-2.40.1
-
mm/vmscan.c | 73 +++++++++++++++++------------------------------------
1 file changed, 23 insertions(+), 50 deletions(-)
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 8f496c2e670a9..f6ce7a1fd78a3 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -4571,13 +4571,12 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+@@ -4571,13 +4571,12 @@ static void lru_gen_age_node(struct pgli
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
{
int i;
struct folio *folio = pfn_folio(pvmw->pfn);
struct mem_cgroup *memcg = folio_memcg(folio);
struct pglist_data *pgdat = folio_pgdat(folio);
-@@ -4594,25 +4593,28 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -4594,25 +4593,28 @@ void lru_gen_look_around(struct page_vma
/* avoid taking the LRU lock under the PTL when possible */
walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL;
for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) {
unsigned long pfn;
-@@ -4637,56 +4639,27 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+@@ -4637,56 +4639,27 @@ void lru_gen_look_around(struct page_vma
!folio_test_swapcache(folio)))
folio_mark_dirty(folio);
}
/******************************************************************************
---
-2.40.1
-
mm/vmscan.c | 112 +++++++++++++++--------------------------
2 files changed, 42 insertions(+), 78 deletions(-)
-diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index 403c7461e7a70..d62a5accf1be4 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -453,18 +453,14 @@ enum {
};
struct lru_gen_mm_walk {
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index f6ce7a1fd78a3..851758303dbf4 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
-@@ -3371,18 +3371,13 @@ void lru_gen_del_mm(struct mm_struct *mm)
+@@ -3371,18 +3371,13 @@ void lru_gen_del_mm(struct mm_struct *mm
if (!lruvec)
continue;
}
list_del_init(&mm->lru_gen.list);
-@@ -3478,68 +3473,54 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk,
+@@ -3478,68 +3473,54 @@ static bool iterate_mm_list(struct lruve
struct mm_struct **iter)
{
bool first = false;
if (*iter || last)
reset_mm_stats(lruvec, walk, last);
-@@ -3567,9 +3548,9 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)
+@@ -3567,9 +3548,9 @@ static bool iterate_mm_list_nowalk(struc
VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
reset_mm_stats(lruvec, NULL, true);
success = true;
-@@ -4172,10 +4153,6 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
+@@ -4172,10 +4153,6 @@ restart:
walk_pmd_range(&val, addr, next, args);
if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
end = (addr | ~PUD_MASK) + 1;
goto done;
-@@ -4208,8 +4185,14 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_
+@@ -4208,8 +4185,14 @@ static void walk_mm(struct lruvec *lruve
walk->next_addr = FIRST_USER_ADDRESS;
do {
/* folio_update_gen() requires stable folio_memcg() */
if (!mem_cgroup_trylock_pages(memcg))
break;
-@@ -4442,25 +4425,12 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+@@ -4442,25 +4425,12 @@ static bool try_to_inc_max_seq(struct lr
success = iterate_mm_list(lruvec, walk, &mm);
if (mm)
walk_mm(lruvec, mm, walk);
}
/******************************************************************************
-@@ -6105,7 +6075,6 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
+@@ -6105,7 +6075,6 @@ void lru_gen_init_lruvec(struct lruvec *
INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
lruvec->mm_state.seq = MIN_NR_GENS;
}
#ifdef CONFIG_MEMCG
-@@ -6138,7 +6107,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+@@ -6138,7 +6107,6 @@ void lru_gen_exit_memcg(struct mem_cgrou
for_each_node(nid) {
struct lruvec *lruvec = get_lruvec(memcg, nid);
VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
sizeof(lruvec->lrugen.nr_pages)));
---
-2.40.1
-
projects / openwrt / staging / stintel.git / commitdiff