mm: vmstats: track TLB flush stats on UP too

The previous patch doing vmstats for TLB flushes ("mm: vmstats: tlb flush
counters") effectively missed UP since arch/x86/mm/tlb.c is only compiled
for SMP.

UP systems do not do remote TLB flushes, so compile those counters out on
UP.

arch/x86/kernel/cpu/mtrr/generic.c calls __flush_tlb() directly.  This is
probably an optimization since both the mtrr code and __flush_tlb() write
cr4.  It would probably be safe to make that a flush_tlb_all() (and then
get these statistics), but the mtrr code is ancient and I'm hesitant to
touch it other than to just stick in the counters.

[akpm@linux-foundation.org: tweak comments]
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index cf512003e6633309587e15f179d0745976f6b510..e6d90babc245c1c2713bea64d436fb2a736b83e6 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -62,6 +62,7 @@ static inline void __flush_tlb_all(void)
 
 static inline void __flush_tlb_one(unsigned long addr)
 {
+       count_vm_event(NR_TLB_LOCAL_FLUSH_ONE);
        __flush_tlb_single(addr);
 }
 
@@ -84,14 +85,38 @@ static inline void __flush_tlb_one(unsigned long addr)
 
 #ifndef CONFIG_SMP
 
-#define flush_tlb() __flush_tlb()
-#define flush_tlb_all() __flush_tlb_all()
-#define local_flush_tlb() __flush_tlb()
+/* "_up" is for UniProcessor.
+ *
+ * This is a helper for other header functions.  *Not* intended to be called
+ * directly.  All global TLB flushes need to either call this, or to bump the
+ * vm statistics themselves.
+ */
+static inline void __flush_tlb_up(void)
+{
+       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
+       __flush_tlb();
+}
+
+static inline void flush_tlb_all(void)
+{
+       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
+       __flush_tlb_all();
+}
+
+static inline void flush_tlb(void)
+{
+       __flush_tlb_up();
+}
+
+static inline void local_flush_tlb(void)
+{
+       __flush_tlb_up();
+}
 
 static inline void flush_tlb_mm(struct mm_struct *mm)
 {
        if (mm == current->active_mm)
-               __flush_tlb();
+               __flush_tlb_up();
 }
 
 static inline void flush_tlb_page(struct vm_area_struct *vma,
@@ -105,14 +130,14 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
                                   unsigned long start, unsigned long end)
 {
        if (vma->vm_mm == current->active_mm)
-               __flush_tlb();
+               __flush_tlb_up();
 }
 
 static inline void flush_tlb_mm_range(struct mm_struct *mm,
           unsigned long start, unsigned long end, unsigned long vmflag)
 {
        if (mm == current->active_mm)
-               __flush_tlb();
+               __flush_tlb_up();
 }
 
 static inline void native_flush_tlb_others(const struct cpumask *cpumask,

diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index d4cdfa67509ec266b787ec7a284e04065bb01c60..ce2d0a2c3e4ff56819152574eefded2e8d64ea69 100644 (file)
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -683,6 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
        }
 
        /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
+       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
        __flush_tlb();
 
        /* Save MTRR state */
@@ -696,6 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
 static void post_set(void) __releases(set_atomicity_lock)
 {
        /* Flush TLBs (no need to flush caches - they are disabled) */
+       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
        __flush_tlb();
 
        /* Intel (P6) standard MTRRs */

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index f030cbe669a57ca2dd0e42b255612628673949cb..ae699b3bbac84a920042349c1fc8605a8f93aba0 100644 (file)
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -280,10 +280,8 @@ static void do_kernel_range_flush(void *info)
        unsigned long addr;
 
        /* flush range by one by one 'invlpg' */
-       for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE) {
-               count_vm_event(NR_TLB_LOCAL_FLUSH_ONE_KERNEL);
+       for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE)
                __flush_tlb_single(addr);
-       }
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index dc2cdf07ac1454731ec0e28282cf3631fc4b517c..1855f0a22add848202b693a16d103072a8f3487a 100644 (file)
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -70,11 +70,12 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
                THP_ZERO_PAGE_ALLOC,
                THP_ZERO_PAGE_ALLOC_FAILED,
 #endif
+#ifdef CONFIG_SMP
                NR_TLB_REMOTE_FLUSH,    /* cpu tried to flush others' tlbs */
                NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */
+#endif
                NR_TLB_LOCAL_FLUSH_ALL,
                NR_TLB_LOCAL_FLUSH_ONE,
-               NR_TLB_LOCAL_FLUSH_ONE_KERNEL,
                NR_VM_EVENT_ITEMS
 };
 

diff --git a/mm/vmstat.c b/mm/vmstat.c
index 00382c53f58296cfe984fab29752c29e790adeda..ca06e9653827e017d493f4a2ffc84ee3bb89318a 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -817,11 +817,12 @@ const char * const vmstat_text[] = {
        "thp_zero_page_alloc",
        "thp_zero_page_alloc_failed",
 #endif
+#ifdef CONFIG_SMP
        "nr_tlb_remote_flush",
        "nr_tlb_remote_flush_received",
+#endif
        "nr_tlb_local_flush_all",
        "nr_tlb_local_flush_one",
-       "nr_tlb_local_flush_one_kernel",
 
 #endif /* CONFIG_VM_EVENTS_COUNTERS */
 };