tools/perf: Add required memory barriers

To match patch bf378d341e48 ("perf: Fix perf ring buffer memory
ordering") change userspace to also adhere to the ordering outlined.

Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Cc: Michael Neuling <mikey@neuling.org>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: james.hogan@imgtec.com
Cc: Vince Weaver <vince@deater.net>
Cc: Victor Kaplansky <VICTORK@il.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Anton Blanchard <anton@samba.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Cc: Michael Ellerman <michael@ellerman.id.au>
Link: http://lkml.kernel.org/r/20131030104246.GH16117@laptop.programming.kicks-ass.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/tools/perf/perf.h b/tools/perf/perf.h
index f61c230beec47f909d4c78af578346a0e96af184..6a587e84fdfe2a238904434a8bdbc1edec7c12b8 100644 (file)
--- a/tools/perf/perf.h
+++ b/tools/perf/perf.h
@@ -4,6 +4,8 @@
 #include <asm/unistd.h>
 
 #if defined(__i386__)
+#define mb()           asm volatile("lock; addl $0,0(%%esp)" ::: "memory")
+#define wmb()          asm volatile("lock; addl $0,0(%%esp)" ::: "memory")
 #define rmb()          asm volatile("lock; addl $0,0(%%esp)" ::: "memory")
 #define cpu_relax()    asm volatile("rep; nop" ::: "memory");
 #define CPUINFO_PROC   "model name"
@@ -13,6 +15,8 @@
 #endif
 
 #if defined(__x86_64__)
+#define mb()           asm volatile("mfence" ::: "memory")
+#define wmb()          asm volatile("sfence" ::: "memory")
 #define rmb()          asm volatile("lfence" ::: "memory")
 #define cpu_relax()    asm volatile("rep; nop" ::: "memory");
 #define CPUINFO_PROC   "model name"
@@ -23,45 +27,61 @@
 
 #ifdef __powerpc__
 #include "../../arch/powerpc/include/uapi/asm/unistd.h"
+#define mb()           asm volatile ("sync" ::: "memory")
+#define wmb()          asm volatile ("sync" ::: "memory")
 #define rmb()          asm volatile ("sync" ::: "memory")
-#define cpu_relax()    asm volatile ("" ::: "memory");
 #define CPUINFO_PROC   "cpu"
 #endif
 
 #ifdef __s390__
+#define mb()           asm volatile("bcr 15,0" ::: "memory")
+#define wmb()          asm volatile("bcr 15,0" ::: "memory")
 #define rmb()          asm volatile("bcr 15,0" ::: "memory")
-#define cpu_relax()    asm volatile("" ::: "memory");
 #endif
 
 #ifdef __sh__
 #if defined(__SH4A__) || defined(__SH5__)
+# define mb()          asm volatile("synco" ::: "memory")
+# define wmb()         asm volatile("synco" ::: "memory")
 # define rmb()         asm volatile("synco" ::: "memory")
 #else
+# define mb()          asm volatile("" ::: "memory")
+# define wmb()         asm volatile("" ::: "memory")
 # define rmb()         asm volatile("" ::: "memory")
 #endif
-#define cpu_relax()    asm volatile("" ::: "memory")
 #define CPUINFO_PROC   "cpu type"
 #endif
 
 #ifdef __hppa__
+#define mb()           asm volatile("" ::: "memory")
+#define wmb()          asm volatile("" ::: "memory")
 #define rmb()          asm volatile("" ::: "memory")
-#define cpu_relax()    asm volatile("" ::: "memory");
 #define CPUINFO_PROC   "cpu"
 #endif
 
 #ifdef __sparc__
+#ifdef __LP64__
+#define mb()           asm volatile("ba,pt %%xcc, 1f\n"        \
+                                    "membar #StoreLoad\n"      \
+                                    "1:\n":::"memory")
+#else
+#define mb()           asm volatile("":::"memory")
+#endif
+#define wmb()          asm volatile("":::"memory")
 #define rmb()          asm volatile("":::"memory")
-#define cpu_relax()    asm volatile("":::"memory")
 #define CPUINFO_PROC   "cpu"
 #endif
 
 #ifdef __alpha__
+#define mb()           asm volatile("mb" ::: "memory")
+#define wmb()          asm volatile("wmb" ::: "memory")
 #define rmb()          asm volatile("mb" ::: "memory")
-#define cpu_relax()    asm volatile("" ::: "memory")
 #define CPUINFO_PROC   "cpu model"
 #endif
 
 #ifdef __ia64__
+#define mb()           asm volatile ("mf" ::: "memory")
+#define wmb()          asm volatile ("mf" ::: "memory")
 #define rmb()          asm volatile ("mf" ::: "memory")
 #define cpu_relax()    asm volatile ("hint @pause" ::: "memory")
 #define CPUINFO_PROC   "model name"
@@ -72,40 +92,55 @@
  * Use the __kuser_memory_barrier helper in the CPU helper page. See
  * arch/arm/kernel/entry-armv.S in the kernel source for details.
  */
+#define mb()           ((void(*)(void))0xffff0fa0)()
+#define wmb()          ((void(*)(void))0xffff0fa0)()
 #define rmb()          ((void(*)(void))0xffff0fa0)()
-#define cpu_relax()    asm volatile("":::"memory")
 #define CPUINFO_PROC   "Processor"
 #endif
 
 #ifdef __aarch64__
-#define rmb()          asm volatile("dmb ld" ::: "memory")
+#define mb()           asm volatile("dmb ish" ::: "memory")
+#define wmb()          asm volatile("dmb ishld" ::: "memory")
+#define rmb()          asm volatile("dmb ishst" ::: "memory")
 #define cpu_relax()    asm volatile("yield" ::: "memory")
 #endif
 
 #ifdef __mips__
-#define rmb()          asm volatile(                                   \
+#define mb()           asm volatile(                                   \
                                ".set   mips2\n\t"                      \
                                "sync\n\t"                              \
                                ".set   mips0"                          \
                                : /* no output */                       \
                                : /* no input */                        \
                                : "memory")
-#define cpu_relax()    asm volatile("" ::: "memory")
+#define wmb()  mb()
+#define rmb()  mb()
 #define CPUINFO_PROC   "cpu model"
 #endif
 
 #ifdef __arc__
+#define mb()           asm volatile("" ::: "memory")
+#define wmb()          asm volatile("" ::: "memory")
 #define rmb()          asm volatile("" ::: "memory")
-#define cpu_relax()    rmb()
 #define CPUINFO_PROC   "Processor"
 #endif
 
 #ifdef __metag__
+#define mb()           asm volatile("" ::: "memory")
+#define wmb()          asm volatile("" ::: "memory")
 #define rmb()          asm volatile("" ::: "memory")
-#define cpu_relax()    asm volatile("" ::: "memory")
 #define CPUINFO_PROC   "CPU"
 #endif
 
+#define barrier() asm volatile ("" ::: "memory")
+
+#ifndef cpu_relax
+#define cpu_relax() barrier()
+#endif
+
+#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
+
+
 #include <time.h>
 #include <unistd.h>
 #include <sys/types.h>

diff --git a/tools/perf/tests/rdpmc.c b/tools/perf/tests/rdpmc.c
index ff94886aad995ec49e267c344885e34079dfdebd..46649c25fa5ed4982c364119792490c4ff861979 100644 (file)
--- a/tools/perf/tests/rdpmc.c
+++ b/tools/perf/tests/rdpmc.c
@@ -9,8 +9,6 @@
 
 #if defined(__x86_64__) || defined(__i386__)
 
-#define barrier() asm volatile("" ::: "memory")
-
 static u64 rdpmc(unsigned int counter)
 {
        unsigned int low, high;

diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h
index e99eaed926827ff4896ad89edbeae441b8bc2f07..ecaa582f40e265b5022c35075da162464df538d2 100644 (file)
--- a/tools/perf/util/evlist.h
+++ b/tools/perf/util/evlist.h
@@ -177,7 +177,7 @@ int perf_evlist__strerror_open(struct perf_evlist *evlist, int err, char *buf, s
 static inline unsigned int perf_mmap__read_head(struct perf_mmap *mm)
 {
        struct perf_event_mmap_page *pc = mm->base;
-       int head = pc->data_head;
+       int head = ACCESS_ONCE(pc->data_head);
        rmb();
        return head;
 }
@@ -190,7 +190,7 @@ static inline void perf_mmap__write_tail(struct perf_mmap *md,
        /*
         * ensure all reads are done before we write the tail out.
         */
-       /* mb(); */
+       mb();
        pc->data_tail = tail;
 }