slub: per cpu partial statistics change

This patch split the cpu_partial_free into 2 parts: cpu_partial_node, PCP refilling
times from node partial; and same name cpu_partial_free, PCP refilling times in
slab_free slow path. A new statistic 'cpu_partial_drain' is added to get PCP
drain to node partial times. These info are useful when do PCP tunning.

The slabinfo.c code is unchanged, since cpu_partial_node is not on slow path.

Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index a32bcfdc783499d1f44f30f01b53468f75be8d56..6388a6681af108a606cbc3a07065ab1ce5840c60 100644 (file)
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -21,7 +21,7 @@ enum stat_item {
        FREE_FROZEN,            /* Freeing to frozen slab */
        FREE_ADD_PARTIAL,       /* Freeing moves slab to partial list */
        FREE_REMOVE_PARTIAL,    /* Freeing removes last object */
-       ALLOC_FROM_PARTIAL,     /* Cpu slab acquired from partial list */
+       ALLOC_FROM_PARTIAL,     /* Cpu slab acquired from node partial list */
        ALLOC_SLAB,             /* Cpu slab acquired from page allocator */
        ALLOC_REFILL,           /* Refill cpu slab from slab freelist */
        ALLOC_NODE_MISMATCH,    /* Switching cpu slab */
@@ -37,7 +37,9 @@ enum stat_item {
        CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */
        CMPXCHG_DOUBLE_FAIL,    /* Number of times that cmpxchg double did not match */
        CPU_PARTIAL_ALLOC,      /* Used cpu partial on alloc */
-       CPU_PARTIAL_FREE,       /* USed cpu partial on free */
+       CPU_PARTIAL_FREE,       /* Refill cpu partial on free */
+       CPU_PARTIAL_NODE,       /* Refill cpu partial from node partial */
+       CPU_PARTIAL_DRAIN,      /* Drain cpu partial to node partial */
        NR_SLUB_STAT_ITEMS };
 
 struct kmem_cache_cpu {

diff --git a/mm/slub.c b/mm/slub.c
index b6666eb3d9c4a1425642e24019341945f6e495c3..24132edcfe3389e894ed7fdd89efed7dd19ea2eb 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1566,6 +1566,7 @@ static void *get_partial_node(struct kmem_cache *s,
                } else {
                        page->freelist = t;
                        available = put_cpu_partial(s, page, 0);
+                       stat(s, CPU_PARTIAL_NODE);
                }
                if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
                        break;
@@ -1979,6 +1980,7 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
                                local_irq_restore(flags);
                                pobjects = 0;
                                pages = 0;
+                               stat(s, CPU_PARTIAL_DRAIN);
                        }
                }
 
@@ -1990,7 +1992,6 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
                page->next = oldpage;
 
        } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
-       stat(s, CPU_PARTIAL_FREE);
        return pobjects;
 }
 
@@ -2474,9 +2475,10 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
                 * If we just froze the page then put it onto the
                 * per cpu partial list.
                 */
-               if (new.frozen && !was_frozen)
+               if (new.frozen && !was_frozen) {
                        put_cpu_partial(s, page, 1);
-
+                       stat(s, CPU_PARTIAL_FREE);
+               }
                /*
                 * The list lock was not taken therefore no list
                 * activity can be necessary.
@@ -5069,6 +5071,8 @@ STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
 STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
 STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
+STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
+STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
 #endif
 
 static struct attribute *slab_attrs[] = {
@@ -5134,6 +5138,8 @@ static struct attribute *slab_attrs[] = {
        &cmpxchg_double_cpu_fail_attr.attr,
        &cpu_partial_alloc_attr.attr,
        &cpu_partial_free_attr.attr,
+       &cpu_partial_node_attr.attr,
+       &cpu_partial_drain_attr.attr,
 #endif
 #ifdef CONFIG_FAILSLAB
        &failslab_attr.attr,