x86/mm: Rename flush_tlb_single() and flush_tlb_one() to __flush_tlb_one_[user|kernel]()

flush_tlb_single() and flush_tlb_one() sound almost identical, but
they really mean "flush one user translation" and "flush one kernel
translation".  Rename them to flush_tlb_one_user() and
flush_tlb_one_kernel() to make the semantics more obvious.

[ I was looking at some PTI-related code, and the flush-one-address code
  is unnecessarily hard to understand because the names of the helpers are
  uninformative.  This came up during PTI review, but no one got around to
  doing it. ]

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Eduardo Valentin <eduval@amazon.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Kees Cook <keescook@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Linux-MM <linux-mm@kvack.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will.deacon@arm.com>
Link: http://lkml.kernel.org/r/3303b02e3c3d049dc5235d5651e0ae6d29a34354.1517414378.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 892df375b6155a51f584760efb9f9e77c3f732e8..554841fab717aef09d2b5cc57410a6eed8c2df0c 100644 (file)
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -297,9 +297,9 @@ static inline void __flush_tlb_global(void)
 {
        PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
 }
-static inline void __flush_tlb_single(unsigned long addr)
+static inline void __flush_tlb_one_user(unsigned long addr)
 {
-       PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
+       PVOP_VCALL1(pv_mmu_ops.flush_tlb_one_user, addr);
 }
 
 static inline void flush_tlb_others(const struct cpumask *cpumask,

diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 6ec54d01972dcf7d79e75bcfbecc84a1f2da80ea..f624f1f10316c248911585f757ea5bd257e98434 100644 (file)
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -217,7 +217,7 @@ struct pv_mmu_ops {
        /* TLB operations */
        void (*flush_tlb_user)(void);
        void (*flush_tlb_kernel)(void);
-       void (*flush_tlb_single)(unsigned long addr);
+       void (*flush_tlb_one_user)(unsigned long addr);
        void (*flush_tlb_others)(const struct cpumask *cpus,
                                 const struct flush_tlb_info *info);
 

diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h
index e67c0620aec2a268537b46d1da80ce6d0ef174a5..e55466760ff8e031433132eab676535e614e224f 100644 (file)
--- a/arch/x86/include/asm/pgtable_32.h
+++ b/arch/x86/include/asm/pgtable_32.h
@@ -61,7 +61,7 @@ void paging_init(void);
 #define kpte_clear_flush(ptep, vaddr)          \
 do {                                           \
        pte_clear(&init_mm, (vaddr), (ptep));   \
-       __flush_tlb_one((vaddr));               \
+       __flush_tlb_one_kernel((vaddr));                \
 } while (0)
 
 #endif /* !__ASSEMBLY__ */

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 2b8f18ca58747ae40b515c2bf674f8faced147e8..84137c22fdfade9bc8224c317808b311ec6f3007 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -140,7 +140,7 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
 #else
 #define __flush_tlb() __native_flush_tlb()
 #define __flush_tlb_global() __native_flush_tlb_global()
-#define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
+#define __flush_tlb_one_user(addr) __native_flush_tlb_one_user(addr)
 #endif
 
 static inline bool tlb_defer_switch_to_init_mm(void)
@@ -400,7 +400,7 @@ static inline void __native_flush_tlb_global(void)
 /*
  * flush one page in the user mapping
  */
-static inline void __native_flush_tlb_single(unsigned long addr)
+static inline void __native_flush_tlb_one_user(unsigned long addr)
 {
        u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
 
@@ -437,18 +437,31 @@ static inline void __flush_tlb_all(void)
 /*
  * flush one page in the kernel mapping
  */
-static inline void __flush_tlb_one(unsigned long addr)
+static inline void __flush_tlb_one_kernel(unsigned long addr)
 {
        count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
-       __flush_tlb_single(addr);
+
+       /*
+        * If PTI is off, then __flush_tlb_one_user() is just INVLPG or its
+        * paravirt equivalent.  Even with PCID, this is sufficient: we only
+        * use PCID if we also use global PTEs for the kernel mapping, and
+        * INVLPG flushes global translations across all address spaces.
+        *
+        * If PTI is on, then the kernel is mapped with non-global PTEs, and
+        * __flush_tlb_one_user() will flush the given address for the current
+        * kernel address space and for its usermode counterpart, but it does
+        * not flush it for other address spaces.
+        */
+       __flush_tlb_one_user(addr);
 
        if (!static_cpu_has(X86_FEATURE_PTI))
                return;
 
        /*
-        * __flush_tlb_single() will have cleared the TLB entry for this ASID,
-        * but since kernel space is replicated across all, we must also
-        * invalidate all others.
+        * See above.  We need to propagate the flush to all other address
+        * spaces.  In principle, we only need to propagate it to kernelmode
+        * address spaces, but the extra bookkeeping we would need is not
+        * worth it.
         */
        invalidate_other_asid();
 }

diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 041096bdef860d356d58873e3e0483384ee98301..99dc79e76bdc5497c8e07c6ee32e74ffe04ff492 100644 (file)
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -200,9 +200,9 @@ static void native_flush_tlb_global(void)
        __native_flush_tlb_global();
 }
 
-static void native_flush_tlb_single(unsigned long addr)
+static void native_flush_tlb_one_user(unsigned long addr)
 {
-       __native_flush_tlb_single(addr);
+       __native_flush_tlb_one_user(addr);
 }
 
 struct static_key paravirt_steal_enabled;
@@ -401,7 +401,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_init = {
 
        .flush_tlb_user = native_flush_tlb,
        .flush_tlb_kernel = native_flush_tlb_global,
-       .flush_tlb_single = native_flush_tlb_single,
+       .flush_tlb_one_user = native_flush_tlb_one_user,
        .flush_tlb_others = native_flush_tlb_others,
 
        .pgd_alloc = __paravirt_pgd_alloc,

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 4a837289f2add8ac3548e68639f38f845ceee6a9..60ae1fe3609fcf802b9396a568e0d6755ae5b983 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -256,7 +256,7 @@ static void __set_pte_vaddr(pud_t *pud, unsigned long vaddr, pte_t new_pte)
         * It's enough to flush this one mapping.
         * (PGE mappings get flushed as well)
         */
-       __flush_tlb_one(vaddr);
+       __flush_tlb_one_kernel(vaddr);
 }
 
 void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte)

diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index c45b6ec5357bcd2e9f6626bd738c700cccd0a173..e2db83bebc3b71a7842404b938f3bb1c2f1e7fae 100644 (file)
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -820,5 +820,5 @@ void __init __early_set_fixmap(enum fixed_addresses idx,
                set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
        else
                pte_clear(&init_mm, addr, pte);
-       __flush_tlb_one(addr);
+       __flush_tlb_one_kernel(addr);
 }

diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
index 58477ec3d66d08acf07c1bc21bb9a55a78fcaa28..7c868670963617865d2e9fb494a08b04a82b49cb 100644 (file)
--- a/arch/x86/mm/kmmio.c
+++ b/arch/x86/mm/kmmio.c
@@ -168,7 +168,7 @@ static int clear_page_presence(struct kmmio_fault_page *f, bool clear)
                return -1;
        }
 
-       __flush_tlb_one(f->addr);
+       __flush_tlb_one_kernel(f->addr);
        return 0;
 }
 

diff --git a/arch/x86/mm/pgtable_32.c b/arch/x86/mm/pgtable_32.c
index c3c5274410a908e762aed936406006d63c3116ac..9bb7f0ab9fe625b77c64bec89b2a0ff9cac222bf 100644 (file)
--- a/arch/x86/mm/pgtable_32.c
+++ b/arch/x86/mm/pgtable_32.c
@@ -63,7 +63,7 @@ void set_pte_vaddr(unsigned long vaddr, pte_t pteval)
         * It's enough to flush this one mapping.
         * (PGE mappings get flushed as well)
         */
-       __flush_tlb_one(vaddr);
+       __flush_tlb_one_kernel(vaddr);
 }
 
 unsigned long __FIXADDR_TOP = 0xfffff000;

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 012d0262484894c3a2e9ee4590f41e9f5756aa3d..0c936435ea9398727da162fd76c5ee4dc4a91892 100644 (file)
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -492,7 +492,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
         *    flush that changes context.tlb_gen from 2 to 3.  If they get
         *    processed on this CPU in reverse order, we'll see
         *     local_tlb_gen == 1, mm_tlb_gen == 3, and end != TLB_FLUSH_ALL.
-        *    If we were to use __flush_tlb_single() and set local_tlb_gen to
+        *    If we were to use __flush_tlb_one_user() and set local_tlb_gen to
         *    3, we'd be break the invariant: we'd update local_tlb_gen above
         *    1 without the full flush that's needed for tlb_gen 2.
         *
@@ -513,7 +513,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
 
                addr = f->start;
                while (addr < f->end) {
-                       __flush_tlb_single(addr);
+                       __flush_tlb_one_user(addr);
                        addr += PAGE_SIZE;
                }
                if (local)
@@ -660,7 +660,7 @@ static void do_kernel_range_flush(void *info)
 
        /* flush range by one by one 'invlpg' */
        for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
-               __flush_tlb_one(addr);
+               __flush_tlb_one_kernel(addr);
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)

diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 8538a6723171a5606058a8823ed1cbb2d343fdb6..7d5d53f36a7abd90754edd64ce3b8edeb5651382 100644 (file)
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -299,7 +299,7 @@ static void bau_process_message(struct msg_desc *mdp, struct bau_control *bcp,
                local_flush_tlb();
                stat->d_alltlb++;
        } else {
-               __flush_tlb_single(msg->address);
+               __flush_tlb_one_user(msg->address);
                stat->d_onetlb++;
        }
        stat->d_requestee++;

diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index d85076223a696d0b00bee7c22a9e28b1e66dc975..aae88fec9941a6dd4183ff2e85f43036e0c2aa88 100644 (file)
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -1300,12 +1300,12 @@ static void xen_flush_tlb(void)
        preempt_enable();
 }
 
-static void xen_flush_tlb_single(unsigned long addr)
+static void xen_flush_tlb_one_user(unsigned long addr)
 {
        struct mmuext_op *op;
        struct multicall_space mcs;
 
-       trace_xen_mmu_flush_tlb_single(addr);
+       trace_xen_mmu_flush_tlb_one_user(addr);
 
        preempt_disable();
 
@@ -2370,7 +2370,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
 
        .flush_tlb_user = xen_flush_tlb,
        .flush_tlb_kernel = xen_flush_tlb,
-       .flush_tlb_single = xen_flush_tlb_single,
+       .flush_tlb_one_user = xen_flush_tlb_one_user,
        .flush_tlb_others = xen_flush_tlb_others,
 
        .pgd_alloc = xen_pgd_alloc,

diff --git a/include/trace/events/xen.h b/include/trace/events/xen.h
index b8adf05c534e725d1e0e3b614181b008e2488f65..7dd8f34c37dfea26f8ec460d3937a46c8f109f4d 100644 (file)
--- a/include/trace/events/xen.h
+++ b/include/trace/events/xen.h
@@ -368,7 +368,7 @@ TRACE_EVENT(xen_mmu_flush_tlb,
            TP_printk("%s", "")
        );
 
-TRACE_EVENT(xen_mmu_flush_tlb_single,
+TRACE_EVENT(xen_mmu_flush_tlb_one_user,
            TP_PROTO(unsigned long addr),
            TP_ARGS(addr),
            TP_STRUCT__entry(