x86/kvm/vmx: Defer TR reload after VM exit

Intel's VMX is daft and resets the hidden TSS limit register to 0x67
on VMX reload, and the 0x67 is not configurable.  KVM currently
reloads TR using the LTR instruction on every exit, but this is quite
slow because LTR is serializing.

The 0x67 limit is entirely harmless unless ioperm() is in use, so
defer the reload until a task using ioperm() is actually running.

Here's some poorly done benchmarking using kvm-unit-tests:

Before:

cpuid 1313
vmcall 1195
mov_from_cr8 11
mov_to_cr8 17
inl_from_pmtimer 6770
inl_from_qemu 6856
inl_from_kernel 2435
outl_to_kernel 1402

After:

cpuid 1291
vmcall 1181
mov_from_cr8 11
mov_to_cr8 16
inl_from_pmtimer 6457
inl_from_qemu 6209
inl_from_kernel 2339
outl_to_kernel 1391

Signed-off-by: Andy Lutomirski <luto@kernel.org>
[Force-reload TR in invalidate_tss_limit. - Paolo]
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index 2e781bcc5e12d2116af99a10ad02e90edff247f3..cb8f9149f6c852377b34bae742bbd54a3de53cf8 100644 (file)
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -205,6 +205,54 @@ static inline void native_load_tr_desc(void)
        asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
 }
 
+static inline void force_reload_TR(void)
+{
+       struct desc_struct *d = get_cpu_gdt_table(smp_processor_id());
+       tss_desc tss;
+
+       memcpy(&tss, &d[GDT_ENTRY_TSS], sizeof(tss_desc));
+
+       /*
+        * LTR requires an available TSS, and the TSS is currently
+        * busy.  Make it be available so that LTR will work.
+        */
+       tss.type = DESC_TSS;
+       write_gdt_entry(d, GDT_ENTRY_TSS, &tss, DESC_TSS);
+
+       load_TR_desc();
+}
+
+DECLARE_PER_CPU(bool, need_tr_refresh);
+
+static inline void refresh_TR(void)
+{
+       DEBUG_LOCKS_WARN_ON(preemptible());
+
+       if (unlikely(this_cpu_read(need_tr_refresh))) {
+               force_reload_TR();
+               this_cpu_write(need_tr_refresh, false);
+       }
+}
+
+/*
+ * If you do something evil that corrupts the cached TSS limit (I'm looking
+ * at you, VMX exits), call this function.
+ *
+ * The optimization here is that the TSS limit only matters for Linux if the
+ * IO bitmap is in use.  If the TSS limit gets forced to its minimum value,
+ * everything works except that IO bitmap will be ignored and all CPL 3 IO
+ * instructions will #GP, which is exactly what we want for normal tasks.
+ */
+static inline void invalidate_tss_limit(void)
+{
+       DEBUG_LOCKS_WARN_ON(preemptible());
+
+       if (unlikely(test_thread_flag(TIF_IO_BITMAP)))
+               force_reload_TR();
+       else
+               this_cpu_write(need_tr_refresh, true);
+}
+
 static inline void native_load_gdt(const struct desc_ptr *dtr)
 {
        asm volatile("lgdt %0"::"m" (*dtr));

diff --git a/arch/x86/kernel/ioport.c b/arch/x86/kernel/ioport.c
index 589b3193f1020b8389a5f64b296cff531e85e849..b01bc851745048f7bdcbf8c9fb1b9fcc383b1b28 100644 (file)
--- a/arch/x86/kernel/ioport.c
+++ b/arch/x86/kernel/ioport.c
@@ -16,6 +16,7 @@
 #include <linux/syscalls.h>
 #include <linux/bitmap.h>
 #include <asm/syscalls.h>
+#include <asm/desc.h>
 
 /*
  * this changes the io permissions bitmap in the current task.
@@ -45,6 +46,10 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on)
                memset(bitmap, 0xff, IO_BITMAP_BYTES);
                t->io_bitmap_ptr = bitmap;
                set_thread_flag(TIF_IO_BITMAP);
+
+               preempt_disable();
+               refresh_TR();
+               preempt_enable();
        }
 
        /*

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index b615a1113f58241bb4dcf0b5cc544b743df24752..7780efa635b911cff705e0bd73abb6064a3e8f62 100644 (file)
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -32,6 +32,7 @@
 #include <asm/mce.h>
 #include <asm/vm86.h>
 #include <asm/switch_to.h>
+#include <asm/desc.h>
 
 /*
  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
@@ -64,6 +65,9 @@ __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
 };
 EXPORT_PER_CPU_SYMBOL(cpu_tss);
 
+DEFINE_PER_CPU(bool, need_tr_refresh);
+EXPORT_PER_CPU_SYMBOL_GPL(need_tr_refresh);
+
 /*
  * this gets called so that we can store lazy state into memory and copy the
  * current task into the new thread.
@@ -209,6 +213,12 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
                 */
                memcpy(tss->io_bitmap, next->io_bitmap_ptr,
                       max(prev->io_bitmap_max, next->io_bitmap_max));
+
+               /*
+                * Make sure that the TSS limit is correct for the CPU
+                * to notice the IO bitmap.
+                */
+               refresh_TR();
        } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
                /*
                 * Clear any possible leftover bits:

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 2dd94cf597ccdbbac1df4945050b3a11ee21b62d..acf6013a0caf91b386762aa6effcbe71cc975da7 100644 (file)
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1990,19 +1990,6 @@ static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
        m->host[i].value = host_val;
 }
 
-static void reload_tss(void)
-{
-       /*
-        * VT restores TR but not its size.  Useless.
-        */
-       struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
-       struct desc_struct *descs;
-
-       descs = (void *)gdt->address;
-       descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
-       load_TR_desc();
-}
-
 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
 {
        u64 guest_efer = vmx->vcpu.arch.efer;
@@ -2172,7 +2159,7 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)
                loadsegment(es, vmx->host_state.es_sel);
        }
 #endif
-       reload_tss();
+       invalidate_tss_limit();
 #ifdef CONFIG_X86_64
        wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
 #endif
@@ -2293,6 +2280,14 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
                            (unsigned long)this_cpu_ptr(&cpu_tss));
                vmcs_writel(HOST_GDTR_BASE, gdt->address);
 
+               /*
+                * VM exits change the host TR limit to 0x67 after a VM
+                * exit.  This is okay, since 0x67 covers everything except
+                * the IO bitmap and have have code to handle the IO bitmap
+                * being lost after a VM exit.
+                */
+               BUILD_BUG_ON(IO_BITMAP_OFFSET - 1 != 0x67);
+
                rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
                vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */