projects / openwrt / staging / linusw.git / blob
? search:


65fb0144b1e7c4803dd71c9ff5398db1800689e1
[openwrt/staging/linusw.git] /
1 From 656287d55d9cfc72a4bcd4d9bd098570f12ce409 Mon Sep 17 00:00:00 2001
2 From: Yu Zhao <yuzhao@google.com>
3 Date: Wed, 21 Dec 2022 21:19:00 -0700
4 Subject: [PATCH 02/19] UPSTREAM: mm: multi-gen LRU: rename lrugen->lists[] to
5 lrugen->folios[]
6
7 lru_gen_folio will be chained into per-node lists by the coming
8 lrugen->list.
9
10 Link: https://lkml.kernel.org/r/20221222041905.2431096-3-yuzhao@google.com
11 Signed-off-by: Yu Zhao <yuzhao@google.com>
12 Cc: Johannes Weiner <hannes@cmpxchg.org>
13 Cc: Jonathan Corbet <corbet@lwn.net>
14 Cc: Michael Larabel <Michael@MichaelLarabel.com>
15 Cc: Michal Hocko <mhocko@kernel.org>
16 Cc: Mike Rapoport <rppt@kernel.org>
17 Cc: Roman Gushchin <roman.gushchin@linux.dev>
18 Cc: Suren Baghdasaryan <surenb@google.com>
19 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
20 Bug: 274865848
21 (cherry picked from commit 6df1b2212950aae2b2188c6645ea18e2a9e3fdd5)
22 Change-Id: I09f53e0fb2cd6b8b3adbb8a80b15dc5efbeae857
23 Signed-off-by: T.J. Mercier <tjmercier@google.com>
24 ---
25 Documentation/mm/multigen_lru.rst | 8 ++++----
26 include/linux/mm_inline.h | 4 ++--
27 include/linux/mmzone.h | 8 ++++----
28 mm/vmscan.c | 20 ++++++++++----------
29 4 files changed, 20 insertions(+), 20 deletions(-)
30
31 diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
32 index d7062c6a89464..d8f721f98868a 100644
33 --- a/Documentation/mm/multigen_lru.rst
34 +++ b/Documentation/mm/multigen_lru.rst
35 @@ -89,15 +89,15 @@ variables are monotonically increasing.
36
37 Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)``
38 bits in order to fit into the gen counter in ``folio->flags``. Each
39 -truncated generation number is an index to ``lrugen->lists[]``. The
40 +truncated generation number is an index to ``lrugen->folios[]``. The
41 sliding window technique is used to track at least ``MIN_NR_GENS`` and
42 at most ``MAX_NR_GENS`` generations. The gen counter stores a value
43 within ``[1, MAX_NR_GENS]`` while a page is on one of
44 -``lrugen->lists[]``; otherwise it stores zero.
45 +``lrugen->folios[]``; otherwise it stores zero.
46
47 Each generation is divided into multiple tiers. A page accessed ``N``
48 times through file descriptors is in tier ``order_base_2(N)``. Unlike
49 -generations, tiers do not have dedicated ``lrugen->lists[]``. In
50 +generations, tiers do not have dedicated ``lrugen->folios[]``. In
51 contrast to moving across generations, which requires the LRU lock,
52 moving across tiers only involves atomic operations on
53 ``folio->flags`` and therefore has a negligible cost. A feedback loop
54 @@ -127,7 +127,7 @@ page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``.
55 Eviction
56 --------
57 The eviction consumes old generations. Given an ``lruvec``, it
58 -increments ``min_seq`` when ``lrugen->lists[]`` indexed by
59 +increments ``min_seq`` when ``lrugen->folios[]`` indexed by
60 ``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to
61 evict from, it first compares ``min_seq[]`` to select the older type.
62 If both types are equally old, it selects the one whose first tier has
63 diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
64 index f63968bd7de59..da38e3d962e2f 100644
65 --- a/include/linux/mm_inline.h
66 +++ b/include/linux/mm_inline.h
67 @@ -256,9 +256,9 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
68 lru_gen_update_size(lruvec, folio, -1, gen);
69 /* for folio_rotate_reclaimable() */
70 if (reclaiming)
71 - list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]);
72 + list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
73 else
74 - list_add(&folio->lru, &lrugen->lists[gen][type][zone]);
75 + list_add(&folio->lru, &lrugen->folios[gen][type][zone]);
76
77 return true;
78 }
79 diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
80 index bd3e4689f72dc..02e4323744715 100644
81 --- a/include/linux/mmzone.h
82 +++ b/include/linux/mmzone.h
83 @@ -312,7 +312,7 @@ enum lruvec_flags {
84 * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An
85 * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the
86 * corresponding generation. The gen counter in folio->flags stores gen+1 while
87 - * a page is on one of lrugen->lists[]. Otherwise it stores 0.
88 + * a page is on one of lrugen->folios[]. Otherwise it stores 0.
89 *
90 * A page is added to the youngest generation on faulting. The aging needs to
91 * check the accessed bit at least twice before handing this page over to the
92 @@ -324,8 +324,8 @@ enum lruvec_flags {
93 * rest of generations, if they exist, are considered inactive. See
94 * lru_gen_is_active().
95 *
96 - * PG_active is always cleared while a page is on one of lrugen->lists[] so that
97 - * the aging needs not to worry about it. And it's set again when a page
98 + * PG_active is always cleared while a page is on one of lrugen->folios[] so
99 + * that the aging needs not to worry about it. And it's set again when a page
100 * considered active is isolated for non-reclaiming purposes, e.g., migration.
101 * See lru_gen_add_folio() and lru_gen_del_folio().
102 *
103 @@ -412,7 +412,7 @@ struct lru_gen_folio {
104 /* the birth time of each generation in jiffies */
105 unsigned long timestamps[MAX_NR_GENS];
106 /* the multi-gen LRU lists, lazily sorted on eviction */
107 - struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
108 + struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
109 /* the multi-gen LRU sizes, eventually consistent */
110 long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
111 /* the exponential moving average of refaulted */
112 diff --git a/mm/vmscan.c b/mm/vmscan.c
113 index 27142caf284c1..b02fed912f742 100644
114 --- a/mm/vmscan.c
115 +++ b/mm/vmscan.c
116 @@ -4258,7 +4258,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
117
118 /* prevent cold/hot inversion if force_scan is true */
119 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
120 - struct list_head *head = &lrugen->lists[old_gen][type][zone];
121 + struct list_head *head = &lrugen->folios[old_gen][type][zone];
122
123 while (!list_empty(head)) {
124 struct folio *folio = lru_to_folio(head);
125 @@ -4269,7 +4269,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
126 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
127
128 new_gen = folio_inc_gen(lruvec, folio, false);
129 - list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]);
130 + list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
131
132 if (!--remaining)
133 return false;
134 @@ -4297,7 +4297,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
135 gen = lru_gen_from_seq(min_seq[type]);
136
137 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
138 - if (!list_empty(&lrugen->lists[gen][type][zone]))
139 + if (!list_empty(&lrugen->folios[gen][type][zone]))
140 goto next;
141 }
142
143 @@ -4762,7 +4762,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
144
145 /* promoted */
146 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
147 - list_move(&folio->lru, &lrugen->lists[gen][type][zone]);
148 + list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
149 return true;
150 }
151
152 @@ -4771,7 +4771,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
153 int hist = lru_hist_from_seq(lrugen->min_seq[type]);
154
155 gen = folio_inc_gen(lruvec, folio, false);
156 - list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]);
157 + list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
158
159 WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
160 lrugen->protected[hist][type][tier - 1] + delta);
161 @@ -4783,7 +4783,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
162 if (folio_test_locked(folio) || folio_test_writeback(folio) ||
163 (type == LRU_GEN_FILE && folio_test_dirty(folio))) {
164 gen = folio_inc_gen(lruvec, folio, true);
165 - list_move(&folio->lru, &lrugen->lists[gen][type][zone]);
166 + list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
167 return true;
168 }
169
170 @@ -4850,7 +4850,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
171 for (zone = sc->reclaim_idx; zone >= 0; zone--) {
172 LIST_HEAD(moved);
173 int skipped = 0;
174 - struct list_head *head = &lrugen->lists[gen][type][zone];
175 + struct list_head *head = &lrugen->folios[gen][type][zone];
176
177 while (!list_empty(head)) {
178 struct folio *folio = lru_to_folio(head);
179 @@ -5250,7 +5250,7 @@ static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
180 int gen, type, zone;
181
182 for_each_gen_type_zone(gen, type, zone) {
183 - if (!list_empty(&lrugen->lists[gen][type][zone]))
184 + if (!list_empty(&lrugen->folios[gen][type][zone]))
185 return false;
186 }
187 }
188 @@ -5295,7 +5295,7 @@ static bool drain_evictable(struct lruvec *lruvec)
189 int remaining = MAX_LRU_BATCH;
190
191 for_each_gen_type_zone(gen, type, zone) {
192 - struct list_head *head = &lruvec->lrugen.lists[gen][type][zone];
193 + struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
194
195 while (!list_empty(head)) {
196 bool success;
197 @@ -5832,7 +5832,7 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
198 lrugen->timestamps[i] = jiffies;
199
200 for_each_gen_type_zone(gen, type, zone)
201 - INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]);
202 + INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
203
204 lruvec->mm_state.seq = MIN_NR_GENS;
205 init_waitqueue_head(&lruvec->mm_state.wait);
206 --
207 2.40.1
208
Staging tree of Linus Walleij