mm: fix race in kunmap_atomic()

Christoph reported a nice splat which illustrated a race in the new stack
based kmap_atomic implementation.

The problem is that we pop our stack slot before we're completely done
resetting its state -- in particular clearing the PTE (sometimes that's
CONFIG_DEBUG_HIGHMEM).  If an interrupt happens before we actually clear
the PTE used for the last slot, that interrupt can reuse the slot in a
dirty state, which triggers a BUG in kmap_atomic().

Fix this by introducing kmap_atomic_idx() which reports the current slot
index without actually releasing it and use that to find the PTE and delay
the _pop() until after we're completely done.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Reported-by: Christoph Hellwig <hch@infradead.org>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c
index c00f119babbfe59a5d51a87fd5ba13f1e4c0f190..c435fd9e1da95c9fdc9d7fab83b3a42caef1b905 100644 (file)
--- a/arch/arm/mm/highmem.c
+++ b/arch/arm/mm/highmem.c
@@ -89,7 +89,7 @@ void __kunmap_atomic(void *kvaddr)
        int idx, type;
 
        if (kvaddr >= (void *)FIXADDR_START) {
-               type = kmap_atomic_idx_pop();
+               type = kmap_atomic_idx();
                idx = type + KM_TYPE_NR * smp_processor_id();
 
                if (cache_is_vivt())
@@ -101,6 +101,7 @@ void __kunmap_atomic(void *kvaddr)
 #else
                (void) idx;  /* to kill a warning */
 #endif
+               kmap_atomic_idx_pop();
        } else if (vaddr >= PKMAP_ADDR(0) && vaddr < PKMAP_ADDR(LAST_PKMAP)) {
                /* this address was obtained through kmap_high_get() */
                kunmap_high(pte_page(pkmap_page_table[PKMAP_NR(vaddr)]));

diff --git a/arch/frv/mm/highmem.c b/arch/frv/mm/highmem.c
index 61088dcc159432ad6063d60aee452d7866a2ef4f..fd7fcd4c2e3399d844f50fed780093880639cb10 100644 (file)
--- a/arch/frv/mm/highmem.c
+++ b/arch/frv/mm/highmem.c
@@ -68,7 +68,7 @@ EXPORT_SYMBOL(__kmap_atomic);
 
 void __kunmap_atomic(void *kvaddr)
 {
-       int type = kmap_atomic_idx_pop();
+       int type = kmap_atomic_idx();
        switch (type) {
        case 0:         __kunmap_atomic_primary(4, 6);  break;
        case 1:         __kunmap_atomic_primary(5, 7);  break;
@@ -83,6 +83,7 @@ void __kunmap_atomic(void *kvaddr)
        default:
                BUG();
        }
+       kmap_atomic_idx_pop();
        pagefault_enable();
 }
 EXPORT_SYMBOL(__kunmap_atomic);

diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c
index 1e69b1fb4b85c16e1a1c5f07bc6f0e054e5196cc..3634c7ea06ac194de365370829080fa3ddc31858 100644 (file)
--- a/arch/mips/mm/highmem.c
+++ b/arch/mips/mm/highmem.c
@@ -74,7 +74,7 @@ void __kunmap_atomic(void *kvaddr)
                return;
        }
 
-       type = kmap_atomic_idx_pop();
+       type = kmap_atomic_idx();
 #ifdef CONFIG_DEBUG_HIGHMEM
        {
                int idx = type + KM_TYPE_NR * smp_processor_id();
@@ -89,6 +89,7 @@ void __kunmap_atomic(void *kvaddr)
                local_flush_tlb_one(vaddr);
        }
 #endif
+       kmap_atomic_idx_pop();
        pagefault_enable();
 }
 EXPORT_SYMBOL(__kunmap_atomic);

diff --git a/arch/mn10300/include/asm/highmem.h b/arch/mn10300/include/asm/highmem.h
index f577ba2268caadc554d2cb34c5163394a83f901f..e2155e686451f51eba912857995fc9ad79461fbc 100644 (file)
--- a/arch/mn10300/include/asm/highmem.h
+++ b/arch/mn10300/include/asm/highmem.h
@@ -101,7 +101,7 @@ static inline void __kunmap_atomic(unsigned long vaddr)
                return;
        }
 
-       type = kmap_atomic_idx_pop();
+       type = kmap_atomic_idx();
 
 #if HIGHMEM_DEBUG
        {
@@ -119,6 +119,8 @@ static inline void __kunmap_atomic(unsigned long vaddr)
                __flush_tlb_one(vaddr);
        }
 #endif
+
+       kmap_atomic_idx_pop();
        pagefault_enable();
 }
 #endif /* __KERNEL__ */

diff --git a/arch/powerpc/mm/highmem.c b/arch/powerpc/mm/highmem.c
index b0848b462bbceb6678a071889cc774e40f77cafd..e7450bdbe83a9380264fc149c4831b587226cd36 100644 (file)
--- a/arch/powerpc/mm/highmem.c
+++ b/arch/powerpc/mm/highmem.c
@@ -62,7 +62,7 @@ void __kunmap_atomic(void *kvaddr)
                return;
        }
 
-       type = kmap_atomic_idx_pop();
+       type = kmap_atomic_idx();
 
 #ifdef CONFIG_DEBUG_HIGHMEM
        {
@@ -79,6 +79,8 @@ void __kunmap_atomic(void *kvaddr)
                local_flush_tlb_page(NULL, vaddr);
        }
 #endif
+
+       kmap_atomic_idx_pop();
        pagefault_enable();
 }
 EXPORT_SYMBOL(__kunmap_atomic);

diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c
index 5e50c09b7dcea796aceea1b88fbd3c0338955abf..4730eac0747b50491876a0bc7e3c2c348c02b067 100644 (file)
--- a/arch/sparc/mm/highmem.c
+++ b/arch/sparc/mm/highmem.c
@@ -75,7 +75,7 @@ void __kunmap_atomic(void *kvaddr)
                return;
        }
 
-       type = kmap_atomic_idx_pop();
+       type = kmap_atomic_idx();
 
 #ifdef CONFIG_DEBUG_HIGHMEM
        {
@@ -104,6 +104,8 @@ void __kunmap_atomic(void *kvaddr)
 #endif
        }
 #endif
+
+       kmap_atomic_idx_pop();
        pagefault_enable();
 }
 EXPORT_SYMBOL(__kunmap_atomic);

diff --git a/arch/tile/mm/highmem.c b/arch/tile/mm/highmem.c
index 8ef6595e162c1348ca65aa21e349bd2d47b81b42..abb57331cf6e1b25fdeccfd295e25ea7464ebbf5 100644 (file)
--- a/arch/tile/mm/highmem.c
+++ b/arch/tile/mm/highmem.c
@@ -241,7 +241,7 @@ void __kunmap_atomic(void *kvaddr)
                pte_t pteval = *pte;
                int idx, type;
 
-               type = kmap_atomic_idx_pop();
+               type = kmap_atomic_idx();
                idx = type + KM_TYPE_NR*smp_processor_id();
 
                /*
@@ -252,6 +252,7 @@ void __kunmap_atomic(void *kvaddr)
                BUG_ON(!pte_present(pteval) && !pte_migrating(pteval));
                kmap_atomic_unregister(pte_page(pteval), vaddr);
                kpte_clear_flush(pte, vaddr);
+               kmap_atomic_idx_pop();
        } else {
                /* Must be a lowmem page */
                BUG_ON(vaddr < PAGE_OFFSET);

diff --git a/arch/x86/mm/highmem_32.c b/arch/x86/mm/highmem_32.c
index d723e369003cb3658eb60f4e7eed1a94a6a4afda..b49962662101a0cf7361f0035e1b017333efc22a 100644 (file)
--- a/arch/x86/mm/highmem_32.c
+++ b/arch/x86/mm/highmem_32.c
@@ -74,7 +74,7 @@ void __kunmap_atomic(void *kvaddr)
            vaddr <= __fix_to_virt(FIX_KMAP_BEGIN)) {
                int idx, type;
 
-               type = kmap_atomic_idx_pop();
+               type = kmap_atomic_idx();
                idx = type + KM_TYPE_NR * smp_processor_id();
 
 #ifdef CONFIG_DEBUG_HIGHMEM
@@ -87,6 +87,7 @@ void __kunmap_atomic(void *kvaddr)
                 * attributes or becomes a protected page in a hypervisor.
                 */
                kpte_clear_flush(kmap_pte-idx, vaddr);
+               kmap_atomic_idx_pop();
        }
 #ifdef CONFIG_DEBUG_HIGHMEM
        else {

diff --git a/arch/x86/mm/iomap_32.c b/arch/x86/mm/iomap_32.c
index 75a3d7f24a2cc1d635c4820b89a1fd4873698ca9..7b179b499fa30944adfbacf0b19a1e042fd4ebe8 100644 (file)
--- a/arch/x86/mm/iomap_32.c
+++ b/arch/x86/mm/iomap_32.c
@@ -98,7 +98,7 @@ iounmap_atomic(void __iomem *kvaddr)
            vaddr <= __fix_to_virt(FIX_KMAP_BEGIN)) {
                int idx, type;
 
-               type = kmap_atomic_idx_pop();
+               type = kmap_atomic_idx();
                idx = type + KM_TYPE_NR * smp_processor_id();
 
 #ifdef CONFIG_DEBUG_HIGHMEM
@@ -111,6 +111,7 @@ iounmap_atomic(void __iomem *kvaddr)
                 * attributes or becomes a protected page in a hypervisor.
                 */
                kpte_clear_flush(kmap_pte-idx, vaddr);
+               kmap_atomic_idx_pop();
        }
 
        pagefault_enable();

diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index 102f76be90da6105f2bfda4c2a4f23aac657bed4..e9138198e8239d878ba04e069112e45eef0d519a 100644 (file)
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -88,6 +88,11 @@ static inline int kmap_atomic_idx_push(void)
        return idx;
 }
 
+static inline int kmap_atomic_idx(void)
+{
+       return __get_cpu_var(__kmap_atomic_idx) - 1;
+}
+
 static inline int kmap_atomic_idx_pop(void)
 {
        int idx = --__get_cpu_var(__kmap_atomic_idx);