x86/mm: Only use tlb_remove_table() for paravirt

If we don't use paravirt; don't play unnecessary and complicated games
to free page-tables.

Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Rik van Riel <riel@surriel.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index b9d37815176f4a6a0917e58af55270567bb295c3..c5ff296bc5d1252f6eaaf847755a3c88fed52c89 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -180,7 +180,7 @@ config X86
        select HAVE_HARDLOCKUP_DETECTOR_PERF    if PERF_EVENTS && HAVE_PERF_EVENTS_NMI
        select HAVE_PERF_REGS
        select HAVE_PERF_USER_STACK_DUMP
-       select HAVE_RCU_TABLE_FREE
+       select HAVE_RCU_TABLE_FREE              if PARAVIRT
        select HAVE_RCU_TABLE_INVALIDATE        if HAVE_RCU_TABLE_FREE
        select HAVE_REGS_AND_STACK_ACCESS_API
        select HAVE_RELIABLE_STACKTRACE         if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION

diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c
index 1147e1fed7fff45e28e9c18e59e81542d1647d7f..ef5f29f913d7b064f1a086ac0674bebb4c8a845a 100644 (file)
--- a/arch/x86/hyperv/mmu.c
+++ b/arch/x86/hyperv/mmu.c
@@ -9,6 +9,7 @@
 #include <asm/mshyperv.h>
 #include <asm/msr.h>
 #include <asm/tlbflush.h>
+#include <asm/tlb.h>
 
 #define CREATE_TRACE_POINTS
 #include <asm/trace/hyperv.h>
@@ -231,4 +232,5 @@ void hyperv_setup_mmu_ops(void)
 
        pr_info("Using hypercall for remote TLB flush\n");
        pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
+       pv_mmu_ops.tlb_remove_table = tlb_remove_table;
 }

diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index d49bbf4bb5c867ef5a086a15b02882d694a961f6..e375d4266b53e35bbe2383d7633d478f29871c4f 100644 (file)
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -309,6 +309,11 @@ static inline void flush_tlb_others(const struct cpumask *cpumask,
        PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info);
 }
 
+static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table)
+{
+       PVOP_VCALL2(pv_mmu_ops.tlb_remove_table, tlb, table);
+}
+
 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
 {
        return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);

diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 180bc0bff0fbd98195b8e81ce9aa7232dde06ee2..4b75acc23b30bad579578b6988140f55bc88f6b9 100644 (file)
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -54,6 +54,7 @@ struct desc_struct;
 struct task_struct;
 struct cpumask;
 struct flush_tlb_info;
+struct mmu_gather;
 
 /*
  * Wrapper type for pointers to code which uses the non-standard
@@ -222,6 +223,8 @@ struct pv_mmu_ops {
        void (*flush_tlb_others)(const struct cpumask *cpus,
                                 const struct flush_tlb_info *info);
 
+       void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
+
        /* Hooks for allocating and freeing a pagetable top-level */
        int  (*pgd_alloc)(struct mm_struct *mm);
        void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 6690cd3fc8b13ddd4318294a09bae5e57c0073cf..29c9da6c62fc16b8b28bec203eb8982b2f570db5 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -552,6 +552,9 @@ extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);
 #ifndef CONFIG_PARAVIRT
 #define flush_tlb_others(mask, info)   \
        native_flush_tlb_others(mask, info)
+
+#define paravirt_tlb_remove_table(tlb, page) \
+       tlb_remove_page(tlb, (void *)(page))
 #endif
 
 #endif /* _ASM_X86_TLBFLUSH_H */

diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 0f471bd934172e673770936e9903d3c81623a6cb..d9b71924c23c9b939986b2d2ab153ba15b7077c3 100644 (file)
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -45,6 +45,7 @@
 #include <asm/apic.h>
 #include <asm/apicdef.h>
 #include <asm/hypervisor.h>
+#include <asm/tlb.h>
 
 static int kvmapf = 1;
 
@@ -636,8 +637,10 @@ static void __init kvm_guest_init(void)
 
        if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
            !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
-           kvm_para_has_feature(KVM_FEATURE_STEAL_TIME))
+           kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
                pv_mmu_ops.flush_tlb_others = kvm_flush_tlb_others;
+               pv_mmu_ops.tlb_remove_table = tlb_remove_table;
+       }
 
        if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
                apic_set_eoi_write(kvm_guest_apic_eoi_write);

diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 930c88341e4ec875013d99f8c844ecace51088fb..afdb303285f874d481c7de2fbe7fd9f80873a424 100644 (file)
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -41,6 +41,7 @@
 #include <asm/tlbflush.h>
 #include <asm/timer.h>
 #include <asm/special_insns.h>
+#include <asm/tlb.h>
 
 /*
  * nop stub, which must not clobber anything *including the stack* to
@@ -409,6 +410,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_init = {
        .flush_tlb_kernel = native_flush_tlb_global,
        .flush_tlb_one_user = native_flush_tlb_one_user,
        .flush_tlb_others = native_flush_tlb_others,
+       .tlb_remove_table = (void (*)(struct mmu_gather *, void *))tlb_remove_page,
 
        .pgd_alloc = __paravirt_pgd_alloc,
        .pgd_free = paravirt_nop,

diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 3ef095c70ae31636fbecd185eac096ef5858d08b..e848a48117856c8e58dcf313339853563f5c6f7f 100644 (file)
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -63,7 +63,7 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 {
        pgtable_page_dtor(pte);
        paravirt_release_pte(page_to_pfn(pte));
-       tlb_remove_table(tlb, pte);
+       paravirt_tlb_remove_table(tlb, pte);
 }
 
 #if CONFIG_PGTABLE_LEVELS > 2
@@ -79,21 +79,21 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
        tlb->need_flush_all = 1;
 #endif
        pgtable_pmd_page_dtor(page);
-       tlb_remove_table(tlb, page);
+       paravirt_tlb_remove_table(tlb, page);
 }
 
 #if CONFIG_PGTABLE_LEVELS > 3
 void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
 {
        paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
-       tlb_remove_table(tlb, virt_to_page(pud));
+       paravirt_tlb_remove_table(tlb, virt_to_page(pud));
 }
 
 #if CONFIG_PGTABLE_LEVELS > 4
 void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d)
 {
        paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT);
-       tlb_remove_table(tlb, virt_to_page(p4d));
+       paravirt_tlb_remove_table(tlb, virt_to_page(p4d));
 }
 #endif /* CONFIG_PGTABLE_LEVELS > 4 */
 #endif /* CONFIG_PGTABLE_LEVELS > 3 */

diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index 52206ad81e4bcabe5d1173781397fdfa7c6d2b3a..3114b7c769701a16fb2ca32b3f598adb50a2f724 100644 (file)
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -67,6 +67,7 @@
 #include <asm/init.h>
 #include <asm/pat.h>
 #include <asm/smp.h>
+#include <asm/tlb.h>
 
 #include <asm/xen/hypercall.h>
 #include <asm/xen/hypervisor.h>
@@ -2397,6 +2398,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
        .flush_tlb_kernel = xen_flush_tlb,
        .flush_tlb_one_user = xen_flush_tlb_one_user,
        .flush_tlb_others = xen_flush_tlb_others,
+       .tlb_remove_table = tlb_remove_table,
 
        .pgd_alloc = xen_pgd_alloc,
        .pgd_free = xen_pgd_free,