slab: overloading the RCU head over the LRU for RCU free

With build-time size checking, we can overload the RCU head over the LRU
of struct page to free pages of a slab in rcu context. This really help to
implement to overload the struct slab over the struct page and this
eventually reduce memory usage and cache footprint of the SLAB.

Acked-by: Andi Kleen <ak@linux.intel.com>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index faf4b7c1ad12702db919bd03a448741b7e579789..959cb369b197f626f01c4919087a7c33ebeebaf9 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -130,6 +130,9 @@ struct page {
 
                struct list_head list;  /* slobs list of pages */
                struct slab *slab_page; /* slab fields */
+               struct rcu_head rcu_head;       /* Used by SLAB
+                                                * when destroying via RCU
+                                                */
        };
 
        /* Remainder is not double word aligned */

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 6c5cc0ea871348673c2a45253e377c1136402250..caaad51fee1f708e6ced5de4e5a4b8c86abb6006 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -51,7 +51,14 @@
  *  }
  *  rcu_read_unlock();
  *
- * See also the comment on struct slab_rcu in mm/slab.c.
+ * This is useful if we need to approach a kernel structure obliquely,
+ * from its address obtained without the usual locking. We can lock
+ * the structure to stabilize it and check it's still at the given address,
+ * only if we can be sure that the memory has not been meanwhile reused
+ * for some other kind of object (which our subsystem's lock might corrupt).
+ *
+ * rcu_read_lock before reading the address, then rcu_read_unlock after
+ * taking the spinlock within the structure expected at that address.
  */
 #define SLAB_DESTROY_BY_RCU    0x00080000UL    /* Defer freeing slabs to RCU */
 #define SLAB_MEM_SPREAD                0x00100000UL    /* Spread some memory over cpuset */

diff --git a/mm/slab.c b/mm/slab.c
index 7e1aabe2b5d86acc72279b78e548a436ffbdcc13..84c4ed62c10da486e3eec04a99f8f4ec407d6e12 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -188,25 +188,6 @@ typedef unsigned int kmem_bufctl_t;
 #define        BUFCTL_ACTIVE   (((kmem_bufctl_t)(~0U))-2)
 #define        SLAB_LIMIT      (((kmem_bufctl_t)(~0U))-3)
 
-/*
- * struct slab_rcu
- *
- * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
- * arrange for kmem_freepages to be called via RCU.  This is useful if
- * we need to approach a kernel structure obliquely, from its address
- * obtained without the usual locking.  We can lock the structure to
- * stabilize it and check it's still at the given address, only if we
- * can be sure that the memory has not been meanwhile reused for some
- * other kind of object (which our subsystem's lock might corrupt).
- *
- * rcu_read_lock before reading the address, then rcu_read_unlock after
- * taking the spinlock within the structure expected at that address.
- */
-struct slab_rcu {
-       struct rcu_head head;
-       struct page *page;
-};
-
 /*
  * struct slab
  *
@@ -215,14 +196,11 @@ struct slab_rcu {
  * Slabs are chained into three list: fully used, partial, fully free slabs.
  */
 struct slab {
-       union {
-               struct {
-                       struct list_head list;
-                       void *s_mem;            /* including colour offset */
-                       unsigned int inuse;     /* num of objs active in slab */
-                       kmem_bufctl_t free;
-               };
-               struct slab_rcu __slab_cover_slab_rcu;
+       struct {
+               struct list_head list;
+               void *s_mem;            /* including colour offset */
+               unsigned int inuse;     /* num of objs active in slab */
+               kmem_bufctl_t free;
        };
 };
 
@@ -1509,6 +1487,8 @@ void __init kmem_cache_init(void)
 {
        int i;
 
+       BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
+                                       sizeof(struct rcu_head));
        kmem_cache = &kmem_cache_boot;
        setup_node_pointer(kmem_cache);
 
@@ -1822,12 +1802,13 @@ static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
 
 static void kmem_rcu_free(struct rcu_head *head)
 {
-       struct slab_rcu *slab_rcu = (struct slab_rcu *)head;
-       struct kmem_cache *cachep = slab_rcu->page->slab_cache;
+       struct kmem_cache *cachep;
+       struct page *page;
 
-       kmem_freepages(cachep, slab_rcu->page);
-       if (OFF_SLAB(cachep))
-               kmem_cache_free(cachep->slabp_cache, slab_rcu);
+       page = container_of(head, struct page, rcu_head);
+       cachep = page->slab_cache;
+
+       kmem_freepages(cachep, page);
 }
 
 #if DEBUG
@@ -2048,16 +2029,27 @@ static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp)
 
        slab_destroy_debugcheck(cachep, slabp);
        if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
-               struct slab_rcu *slab_rcu;
+               struct rcu_head *head;
+
+               /*
+                * RCU free overloads the RCU head over the LRU.
+                * slab_page has been overloeaded over the LRU,
+                * however it is not used from now on so that
+                * we can use it safely.
+                */
+               head = (void *)&page->rcu_head;
+               call_rcu(head, kmem_rcu_free);
 
-               slab_rcu = (struct slab_rcu *)slabp;
-               slab_rcu->page = page;
-               call_rcu(&slab_rcu->head, kmem_rcu_free);
        } else {
                kmem_freepages(cachep, page);
-               if (OFF_SLAB(cachep))
-                       kmem_cache_free(cachep->slabp_cache, slabp);
        }
+
+       /*
+        * From now on, we don't use slab management
+        * although actual page can be freed in rcu context
+        */
+       if (OFF_SLAB(cachep))
+               kmem_cache_free(cachep->slabp_cache, slabp);
 }
 
 /**