[PATCH] Use Zoned VM Counters for NUMA statistics

The numa statistics are really event counters.  But they are per node and
so we have had special treatment for these counters through additional
fields on the pcp structure.  We can now use the per zone nature of the
zoned VM counters to realize these.

This will shrink the size of the pcp structure on NUMA systems.  We will
have some room to add additional per zone counters that will all still fit
in the same cacheline.

 Bits	Prior pcp size	  	Size after patch	We can add
 ------------------------------------------------------------------
 64	128 bytes (16 words)	80 bytes (10 words)	48
 32	 76 bytes (19 words)	56 bytes (14 words)	8 (64 byte cacheline)
							72 (128 byte)

Remove the special statistics for numa and replace them with zoned vm
counters.  This has the side effect that global sums of these events now
show up in /proc/vmstat.

Also take the opportunity to move the zone_statistics() function from
page_alloc.c into vmstat.c.

Discussions:
V2 http://marc.theaimsgroup.com/?t=115048227000002&r=1&w=2

Signed-off-by: Christoph Lameter <clameter@sgi.com>
Acked-by: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/drivers/base/node.c b/drivers/base/node.c
index 772eadac57a72e553cad3402f13443d9f1235b40..d7de1753e09412198db9707cab7dd8484eeab2d8 100644 (file)
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -94,28 +94,6 @@ static SYSDEV_ATTR(meminfo, S_IRUGO, node_read_meminfo, NULL);
 
 static ssize_t node_read_numastat(struct sys_device * dev, char * buf)
 {
-       unsigned long numa_hit, numa_miss, interleave_hit, numa_foreign;
-       unsigned long local_node, other_node;
-       int i, cpu;
-       pg_data_t *pg = NODE_DATA(dev->id);
-       numa_hit = 0;
-       numa_miss = 0;
-       interleave_hit = 0;
-       numa_foreign = 0;
-       local_node = 0;
-       other_node = 0;
-       for (i = 0; i < MAX_NR_ZONES; i++) {
-               struct zone *z = &pg->node_zones[i];
-               for_each_online_cpu(cpu) {
-                       struct per_cpu_pageset *ps = zone_pcp(z,cpu);
-                       numa_hit += ps->numa_hit;
-                       numa_miss += ps->numa_miss;
-                       numa_foreign += ps->numa_foreign;
-                       interleave_hit += ps->interleave_hit;
-                       local_node += ps->local_node;
-                       other_node += ps->other_node;
-               }
-       }
        return sprintf(buf,
                       "numa_hit %lu\n"
                       "numa_miss %lu\n"
@@ -123,12 +101,12 @@ static ssize_t node_read_numastat(struct sys_device * dev, char * buf)
                       "interleave_hit %lu\n"
                       "local_node %lu\n"
                       "other_node %lu\n",
-                      numa_hit,
-                      numa_miss,
-                      numa_foreign,
-                      interleave_hit,
-                      local_node,
-                      other_node);
+                      node_page_state(dev->id, NUMA_HIT),
+                      node_page_state(dev->id, NUMA_MISS),
+                      node_page_state(dev->id, NUMA_FOREIGN),
+                      node_page_state(dev->id, NUMA_INTERLEAVE_HIT),
+                      node_page_state(dev->id, NUMA_LOCAL),
+                      node_page_state(dev->id, NUMA_OTHER));
 }
 static SYSDEV_ATTR(numastat, S_IRUGO, node_read_numastat, NULL);
 

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 2dbeec1d2874fe96a8537d523e4e84276931f660..27e748eb72b0d897f02d327ce3df0cdcd969fbaf 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -57,6 +57,14 @@ enum zone_stat_item {
        NR_WRITEBACK,
        NR_UNSTABLE_NFS,        /* NFS unstable pages */
        NR_BOUNCE,
+#ifdef CONFIG_NUMA
+       NUMA_HIT,               /* allocated in intended node */
+       NUMA_MISS,              /* allocated in non intended node */
+       NUMA_FOREIGN,           /* was intended here, hit elsewhere */
+       NUMA_INTERLEAVE_HIT,    /* interleaver preferred this zone */
+       NUMA_LOCAL,             /* allocation from local node */
+       NUMA_OTHER,             /* allocation from other node */
+#endif
        NR_VM_ZONE_STAT_ITEMS };
 
 struct per_cpu_pages {
@@ -71,15 +79,6 @@ struct per_cpu_pageset {
 #ifdef CONFIG_SMP
        s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
 #endif
-
-#ifdef CONFIG_NUMA
-       unsigned long numa_hit;         /* allocated in intended node */
-       unsigned long numa_miss;        /* allocated in non intended node */
-       unsigned long numa_foreign;     /* was intended here, hit elsewhere */
-       unsigned long interleave_hit;   /* interleaver prefered this zone */
-       unsigned long local_node;       /* allocation from local node */
-       unsigned long other_node;       /* allocation from other node */
-#endif
 } ____cacheline_aligned_in_smp;
 
 #ifdef CONFIG_NUMA

diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 5fad1613e7d6f1e2a46edb2b52df21c18df30618..16173b63ee678131f41d8fa8763586be74c631a5 100644 (file)
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -173,9 +173,15 @@ static inline unsigned long node_page_state(int node,
 #endif
                zone_page_state(&zones[ZONE_DMA], item);
 }
+
+extern void zone_statistics(struct zonelist *, struct zone *);
+
 #else
+
 #define node_page_state(node, item) global_page_state(item)
-#endif
+#define zone_statistics(_zl,_z) do { } while (0)
+
+#endif /* CONFIG_NUMA */
 
 #define __add_zone_page_state(__z, __i, __d)   \
                __mod_zone_page_state(__z, __i, __d)
@@ -190,6 +196,8 @@ static inline void zap_zone_vm_stats(struct zone *zone)
        memset(zone->vm_stat, 0, sizeof(zone->vm_stat));
 }
 
+extern void inc_zone_state(struct zone *, enum zone_stat_item);
+
 #ifdef CONFIG_SMP
 void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
 void __inc_zone_page_state(struct page *, enum zone_stat_item);

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 6b9740bbf4c0199cee2096ad6e6544d4c6b8a255..e07e27e846a22981a05804aefcdbdb31d88f15be 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1209,10 +1209,8 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
 
        zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
        page = __alloc_pages(gfp, order, zl);
-       if (page && page_zone(page) == zl->zones[0]) {
-               zone_pcp(zl->zones[0],get_cpu())->interleave_hit++;
-               put_cpu();
-       }
+       if (page && page_zone(page) == zl->zones[0])
+               inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
        return page;
 }
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6aa2c31f513bc8d8448ed293fe3b2affe6bc1a86..d61671260f924fce3d6d40e98d7b8578d41f739e 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -709,27 +709,6 @@ void drain_local_pages(void)
 }
 #endif /* CONFIG_PM */
 
-static void zone_statistics(struct zonelist *zonelist, struct zone *z, int cpu)
-{
-#ifdef CONFIG_NUMA
-       pg_data_t *pg = z->zone_pgdat;
-       pg_data_t *orig = zonelist->zones[0]->zone_pgdat;
-       struct per_cpu_pageset *p;
-
-       p = zone_pcp(z, cpu);
-       if (pg == orig) {
-               p->numa_hit++;
-       } else {
-               p->numa_miss++;
-               zone_pcp(zonelist->zones[0], cpu)->numa_foreign++;
-       }
-       if (pg == NODE_DATA(numa_node_id()))
-               p->local_node++;
-       else
-               p->other_node++;
-#endif
-}
-
 /*
  * Free a 0-order page
  */
@@ -827,7 +806,7 @@ again:
        }
 
        __mod_page_state_zone(zone, pgalloc, 1 << order);
-       zone_statistics(zonelist, zone, cpu);
+       zone_statistics(zonelist, zone);
        local_irq_restore(flags);
        put_cpu();
 

diff --git a/mm/vmstat.c b/mm/vmstat.c
index 06a6d10521982eace9c47c5fa85f90c644de8998..ee7f896662503039339b750779c243f31a2659df 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -185,9 +185,8 @@ EXPORT_SYMBOL(mod_zone_page_state);
  * in between and therefore the atomicity vs. interrupt cannot be exploited
  * in a useful way here.
  */
-void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
+static void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
 {
-       struct zone *zone = page_zone(page);
        s8 *p = diff_pointer(zone, item);
 
        (*p)++;
@@ -197,6 +196,11 @@ void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
                *p = 0;
        }
 }
+
+void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
+{
+       __inc_zone_state(page_zone(page), item);
+}
 EXPORT_SYMBOL(__inc_zone_page_state);
 
 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
@@ -213,22 +217,23 @@ void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
 }
 EXPORT_SYMBOL(__dec_zone_page_state);
 
+void inc_zone_state(struct zone *zone, enum zone_stat_item item)
+{
+       unsigned long flags;
+
+       local_irq_save(flags);
+       __inc_zone_state(zone, item);
+       local_irq_restore(flags);
+}
+
 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
 {
        unsigned long flags;
        struct zone *zone;
-       s8 *p;
 
        zone = page_zone(page);
        local_irq_save(flags);
-       p = diff_pointer(zone, item);
-
-       (*p)++;
-
-       if (unlikely(*p > STAT_THRESHOLD)) {
-               zone_page_state_add(*p, zone, item);
-               *p = 0;
-       }
+       __inc_zone_state(zone, item);
        local_irq_restore(flags);
 }
 EXPORT_SYMBOL(inc_zone_page_state);
@@ -297,6 +302,28 @@ EXPORT_SYMBOL(refresh_vm_stats);
 
 #endif
 
+#ifdef CONFIG_NUMA
+/*
+ * zonelist = the list of zones passed to the allocator
+ * z       = the zone from which the allocation occurred.
+ *
+ * Must be called with interrupts disabled.
+ */
+void zone_statistics(struct zonelist *zonelist, struct zone *z)
+{
+       if (z->zone_pgdat == zonelist->zones[0]->zone_pgdat) {
+               __inc_zone_state(z, NUMA_HIT);
+       } else {
+               __inc_zone_state(z, NUMA_MISS);
+               __inc_zone_state(zonelist->zones[0], NUMA_FOREIGN);
+       }
+       if (z->zone_pgdat == NODE_DATA(numa_node_id()))
+               __inc_zone_state(z, NUMA_LOCAL);
+       else
+               __inc_zone_state(z, NUMA_OTHER);
+}
+#endif
+
 #ifdef CONFIG_PROC_FS
 
 #include <linux/seq_file.h>
@@ -369,6 +396,15 @@ static char *vmstat_text[] = {
        "nr_unstable",
        "nr_bounce",
 
+#ifdef CONFIG_NUMA
+       "numa_hit",
+       "numa_miss",
+       "numa_foreign",
+       "numa_interleave",
+       "numa_local",
+       "numa_other",
+#endif
+
        /* Event counters */
        "pgpgin",
        "pgpgout",
@@ -490,21 +526,6 @@ static int zoneinfo_show(struct seq_file *m, void *arg)
                                           pageset->pcp[j].high,
                                           pageset->pcp[j].batch);
                        }
-#ifdef CONFIG_NUMA
-                       seq_printf(m,
-                                  "\n            numa_hit:       %lu"
-                                  "\n            numa_miss:      %lu"
-                                  "\n            numa_foreign:   %lu"
-                                  "\n            interleave_hit: %lu"
-                                  "\n            local_node:     %lu"
-                                  "\n            other_node:     %lu",
-                                  pageset->numa_hit,
-                                  pageset->numa_miss,
-                                  pageset->numa_foreign,
-                                  pageset->interleave_hit,
-                                  pageset->local_node,
-                                  pageset->other_node);
-#endif
                }
                seq_printf(m,
                           "\n  all_unreclaimable: %u"