kvm: svm: Use the hardware provided GPA instead of page walk

When a guest causes a NPF which requires emulation, KVM sometimes walks
the guest page tables to translate the GVA to a GPA. This is unnecessary
most of the time on AMD hardware since the hardware provides the GPA in
EXITINFO2.

The only exception cases involve string operations involving rep or
operations that use two memory locations. With rep, the GPA will only be
the value of the initial NPF and with dual memory locations we won't know
which memory address was translated into EXITINFO2.

Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Brijesh Singh <brijesh.singh@amd.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index e9cd7befcb76af6e4052c193b879b5f869420754..3e8c287090e428bd04dfde09df08925b11bd64cd 100644 (file)
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -441,5 +441,6 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq);
 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt);
 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt);
+bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt);
 
 #endif /* _ASM_X86_KVM_X86_EMULATE_H */

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 99a71d90b6ae284318a3b4f0852cd720231ceb61..0419e114f27b05416568466d066ba82dea43f939 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -675,6 +675,9 @@ struct kvm_vcpu_arch {
 
        int pending_ioapic_eoi;
        int pending_external_vector;
+
+       /* GPA available (AMD only) */
+       bool gpa_available;
 };
 
 struct kvm_lpage_info {

diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index 56628a44668b7ec43cee087af817ebb1725cf7cd..2b8349a2b14be0a558710089cb5d688a42ac1465 100644 (file)
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -173,6 +173,7 @@
 #define NearBranch  ((u64)1 << 52)  /* Near branches */
 #define No16       ((u64)1 << 53)  /* No 16 bit operand */
 #define IncSP       ((u64)1 << 54)  /* SP is incremented before ModRM calc */
+#define TwoMemOp    ((u64)1 << 55)  /* Instruction has two memory operand */
 
 #define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)
 
@@ -4256,7 +4257,7 @@ static const struct opcode group1[] = {
 };
 
 static const struct opcode group1A[] = {
-       I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
+       I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
 };
 
 static const struct opcode group2[] = {
@@ -4294,7 +4295,7 @@ static const struct opcode group5[] = {
        I(SrcMemFAddr | ImplicitOps,            em_call_far),
        I(SrcMem | NearBranch,                  em_jmp_abs),
        I(SrcMemFAddr | ImplicitOps,            em_jmp_far),
-       I(SrcMem | Stack,                       em_push), D(Undefined),
+       I(SrcMem | Stack | TwoMemOp,            em_push), D(Undefined),
 };
 
 static const struct opcode group6[] = {
@@ -4514,8 +4515,8 @@ static const struct opcode opcode_table[256] = {
        /* 0xA0 - 0xA7 */
        I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
        I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
-       I2bv(SrcSI | DstDI | Mov | String, em_mov),
-       F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
+       I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
+       F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
        /* 0xA8 - 0xAF */
        F2bv(DstAcc | SrcImm | NoWrite, em_test),
        I2bv(SrcAcc | DstDI | Mov | String, em_mov),
@@ -5629,3 +5630,14 @@ void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
 {
        writeback_registers(ctxt);
 }
+
+bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
+{
+       if (ctxt->rep_prefix && (ctxt->d & String))
+               return false;
+
+       if (ctxt->d & TwoMemOp)
+               return false;
+
+       return true;
+}

diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 08a4d3ab3455734a669ffe0326450f4d0fb34e55..d0414f054bdf7c2f7adcf253cddeb2baba74d446 100644 (file)
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -4182,6 +4182,8 @@ static int handle_exit(struct kvm_vcpu *vcpu)
 
        trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
 
+       vcpu->arch.gpa_available = (exit_code == SVM_EXIT_NPF);
+
        if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
                vcpu->arch.cr0 = svm->vmcb->save.cr0;
        if (npt_enabled)

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c3ee5e29ea2aebf6b684f8f456ce5ce23af60caa..edff19d1df97836a112b57fd4ab260ac3a8c53f4 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4462,6 +4462,21 @@ out:
 }
 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
 
+static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
+                           gpa_t gpa, bool write)
+{
+       /* For APIC access vmexit */
+       if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
+               return 1;
+
+       if (vcpu_match_mmio_gpa(vcpu, gpa)) {
+               trace_vcpu_match_mmio(gva, gpa, write, true);
+               return 1;
+       }
+
+       return 0;
+}
+
 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
                                gpa_t *gpa, struct x86_exception *exception,
                                bool write)
@@ -4488,16 +4503,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
        if (*gpa == UNMAPPED_GVA)
                return -1;
 
-       /* For APIC access vmexit */
-       if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
-               return 1;
-
-       if (vcpu_match_mmio_gpa(vcpu, *gpa)) {
-               trace_vcpu_match_mmio(gva, *gpa, write, true);
-               return 1;
-       }
-
-       return 0;
+       return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
 }
 
 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
@@ -4594,6 +4600,22 @@ static int emulator_read_write_onepage(unsigned long addr, void *val,
        int handled, ret;
        bool write = ops->write;
        struct kvm_mmio_fragment *frag;
+       struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
+
+       /*
+        * If the exit was due to a NPF we may already have a GPA.
+        * If the GPA is present, use it to avoid the GVA to GPA table walk.
+        * Note, this cannot be used on string operations since string
+        * operation using rep will only have the initial GPA from the NPF
+        * occurred.
+        */
+       if (vcpu->arch.gpa_available &&
+           emulator_can_use_gpa(ctxt) &&
+           vcpu_is_mmio_gpa(vcpu, addr, exception->address, write) &&
+           (addr & ~PAGE_MASK) == (exception->address & ~PAGE_MASK)) {
+               gpa = exception->address;
+               goto mmio;
+       }
 
        ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
 
@@ -5610,6 +5632,9 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
        }
 
 restart:
+       /* Save the faulting GPA (cr2) in the address field */
+       ctxt->exception.address = cr2;
+
        r = x86_emulate_insn(ctxt);
 
        if (r == EMULATION_INTERCEPTED)