memcg, slab: merge memcg_{bind,release}_pages to memcg_{un}charge_slab

Currently we have two pairs of kmemcg-related functions that are called on
slab alloc/free.  The first is memcg_{bind,release}_pages that count the
total number of pages allocated on a kmem cache.  The second is
memcg_{un}charge_slab that {un}charge slab pages to kmemcg resource
counter.  Let's just merge them to keep the code clean.

Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Glauber Costa <glommer@gmail.com>
Cc: Pekka Enberg <penberg@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 087a453141810dcbf519d4e6810f9f78adff42cb..d38d190f4cecbfdfb4e8ccfd2e55153d49ff831e 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -506,8 +506,8 @@ void memcg_update_array_size(int num_groups);
 struct kmem_cache *
 __memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp);
 
-int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size);
-void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size);
+int __memcg_charge_slab(struct kmem_cache *cachep, gfp_t gfp, int order);
+void __memcg_uncharge_slab(struct kmem_cache *cachep, int order);
 
 int __kmem_cache_destroy_memcg_children(struct kmem_cache *s);
 

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6b1c45ced7330db7e6502493aa47df7dedd0732f..86a2078805e5ca0d7f3bdeb8f6d803ee285519e4 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2954,7 +2954,7 @@ static int mem_cgroup_slabinfo_read(struct seq_file *m, void *v)
 }
 #endif
 
-int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
+static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
 {
        struct res_counter *fail_res;
        int ret = 0;
@@ -2992,7 +2992,7 @@ int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
        return ret;
 }
 
-void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
+static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
 {
        res_counter_uncharge(&memcg->res, size);
        if (do_swap_account)
@@ -3390,6 +3390,24 @@ static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
        __memcg_create_cache_enqueue(memcg, cachep);
        memcg_resume_kmem_account();
 }
+
+int __memcg_charge_slab(struct kmem_cache *cachep, gfp_t gfp, int order)
+{
+       int res;
+
+       res = memcg_charge_kmem(cachep->memcg_params->memcg, gfp,
+                               PAGE_SIZE << order);
+       if (!res)
+               atomic_add(1 << order, &cachep->memcg_params->nr_pages);
+       return res;
+}
+
+void __memcg_uncharge_slab(struct kmem_cache *cachep, int order)
+{
+       memcg_uncharge_kmem(cachep->memcg_params->memcg, PAGE_SIZE << order);
+       atomic_sub(1 << order, &cachep->memcg_params->nr_pages);
+}
+
 /*
  * Return the kmem_cache we're supposed to use for a slab allocation.
  * We try to use the current memcg's version of the cache.

diff --git a/mm/slab.c b/mm/slab.c
index 7067ea7f3927da94942da915bd4a9f78eb8bbe90..9ca3b87edabc699a7da3ceea71b61ee34a300e03 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -1712,7 +1712,6 @@ static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
        __SetPageSlab(page);
        if (page->pfmemalloc)
                SetPageSlabPfmemalloc(page);
-       memcg_bind_pages(cachep, cachep->gfporder);
 
        if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
                kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
@@ -1748,7 +1747,6 @@ static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
        page_mapcount_reset(page);
        page->mapping = NULL;
 
-       memcg_release_pages(cachep, cachep->gfporder);
        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += nr_freed;
        __free_pages(page, cachep->gfporder);

diff --git a/mm/slab.h b/mm/slab.h
index b59447ac45337329612bda515f2742583623e0f2..961a3fb1f5a2c69454e123e5a2d7c34b652c2a1c 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -121,18 +121,6 @@ static inline bool is_root_cache(struct kmem_cache *s)
        return !s->memcg_params || s->memcg_params->is_root_cache;
 }
 
-static inline void memcg_bind_pages(struct kmem_cache *s, int order)
-{
-       if (!is_root_cache(s))
-               atomic_add(1 << order, &s->memcg_params->nr_pages);
-}
-
-static inline void memcg_release_pages(struct kmem_cache *s, int order)
-{
-       if (!is_root_cache(s))
-               atomic_sub(1 << order, &s->memcg_params->nr_pages);
-}
-
 static inline bool slab_equal_or_root(struct kmem_cache *s,
                                        struct kmem_cache *p)
 {
@@ -198,8 +186,7 @@ static __always_inline int memcg_charge_slab(struct kmem_cache *s,
                return 0;
        if (is_root_cache(s))
                return 0;
-       return memcg_charge_kmem(s->memcg_params->memcg, gfp,
-                                PAGE_SIZE << order);
+       return __memcg_charge_slab(s, gfp, order);
 }
 
 static __always_inline void memcg_uncharge_slab(struct kmem_cache *s, int order)
@@ -208,7 +195,7 @@ static __always_inline void memcg_uncharge_slab(struct kmem_cache *s, int order)
                return;
        if (is_root_cache(s))
                return;
-       memcg_uncharge_kmem(s->memcg_params->memcg, PAGE_SIZE << order);
+       __memcg_uncharge_slab(s, order);
 }
 #else
 static inline bool is_root_cache(struct kmem_cache *s)
@@ -216,14 +203,6 @@ static inline bool is_root_cache(struct kmem_cache *s)
        return true;
 }
 
-static inline void memcg_bind_pages(struct kmem_cache *s, int order)
-{
-}
-
-static inline void memcg_release_pages(struct kmem_cache *s, int order)
-{
-}
-
 static inline bool slab_equal_or_root(struct kmem_cache *s,
                                      struct kmem_cache *p)
 {

diff --git a/mm/slub.c b/mm/slub.c
index 5d1b653183abbfdfb3c0678cba0bf582bb08089f..9e288d7c5e6afc8996d789f3c3e7734502cd3bac 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1422,7 +1422,6 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
 
        order = compound_order(page);
        inc_slabs_node(s, page_to_nid(page), page->objects);
-       memcg_bind_pages(s, order);
        page->slab_cache = s;
        __SetPageSlab(page);
        if (page->pfmemalloc)
@@ -1473,7 +1472,6 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
        __ClearPageSlabPfmemalloc(page);
        __ClearPageSlab(page);
 
-       memcg_release_pages(s, order);
        page_mapcount_reset(page);
        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += pages;