mmiowb: Hook up mmiowb helpers to spinlocks and generic I/O accessors

Removing explicit calls to mmiowb() from driver code means that we must
now call into the generic mmiowb_spin_{lock,unlock}() functions from the
core spinlock code. In order to elide barriers following critical
sections without any I/O writes, we also hook into the asm-generic I/O
routines.

Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h
index 303871651f8aa0cd92dff49cb188c2ebd5e0fab5..bc490a7466021451cf894de31df9709cc3e3bdf6 100644 (file)
--- a/include/asm-generic/io.h
+++ b/include/asm-generic/io.h
@@ -19,6 +19,7 @@
 #include <asm-generic/iomap.h>
 #endif
 
+#include <asm/mmiowb.h>
 #include <asm-generic/pci_iomap.h>
 
 #ifndef mmiowb
@@ -49,7 +50,7 @@
 
 /* serialize device access against a spin_unlock, usually handled there. */
 #ifndef __io_aw
-#define __io_aw()      barrier()
+#define __io_aw()      mmiowb_set_pending()
 #endif
 
 #ifndef __io_pbw

diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index e089157dcf97cd6cc44eb2bbfb7eca8afd054e03..ed7c4d6b8235f6c50e1847a7259cabdde5247b97 100644 (file)
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -57,6 +57,7 @@
 #include <linux/stringify.h>
 #include <linux/bottom_half.h>
 #include <asm/barrier.h>
+#include <asm/mmiowb.h>
 
 
 /*
@@ -178,6 +179,7 @@ static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock)
 {
        __acquire(lock);
        arch_spin_lock(&lock->raw_lock);
+       mmiowb_spin_lock();
 }
 
 #ifndef arch_spin_lock_flags
@@ -189,15 +191,22 @@ do_raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long *flags) __acquires(lo
 {
        __acquire(lock);
        arch_spin_lock_flags(&lock->raw_lock, *flags);
+       mmiowb_spin_lock();
 }
 
 static inline int do_raw_spin_trylock(raw_spinlock_t *lock)
 {
-       return arch_spin_trylock(&(lock)->raw_lock);
+       int ret = arch_spin_trylock(&(lock)->raw_lock);
+
+       if (ret)
+               mmiowb_spin_lock();
+
+       return ret;
 }
 
 static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 {
+       mmiowb_spin_unlock();
        arch_spin_unlock(&lock->raw_lock);
        __release(lock);
 }

diff --git a/kernel/locking/spinlock_debug.c b/kernel/locking/spinlock_debug.c
index 9aa0fccd5d432d7383bbf186d0cd7fc42893108d..399669f7eba8e9569830e1c35b7f37c3c712025b 100644 (file)
--- a/kernel/locking/spinlock_debug.c
+++ b/kernel/locking/spinlock_debug.c
@@ -111,6 +111,7 @@ void do_raw_spin_lock(raw_spinlock_t *lock)
 {
        debug_spin_lock_before(lock);
        arch_spin_lock(&lock->raw_lock);
+       mmiowb_spin_lock();
        debug_spin_lock_after(lock);
 }
 
@@ -118,8 +119,10 @@ int do_raw_spin_trylock(raw_spinlock_t *lock)
 {
        int ret = arch_spin_trylock(&lock->raw_lock);
 
-       if (ret)
+       if (ret) {
+               mmiowb_spin_lock();
                debug_spin_lock_after(lock);
+       }
 #ifndef CONFIG_SMP
        /*
         * Must not happen on UP:
@@ -131,6 +134,7 @@ int do_raw_spin_trylock(raw_spinlock_t *lock)
 
 void do_raw_spin_unlock(raw_spinlock_t *lock)
 {
+       mmiowb_spin_unlock();
        debug_spin_unlock(lock);
        arch_spin_unlock(&lock->raw_lock);
 }