x86/io: Unroll string I/O when SEV is active

Secure Encrypted Virtualization (SEV) does not support string I/O, so
unroll the string I/O operation into a loop operating on one element at
a time.

[ tglx: Gave the static key a real name instead of the obscure __sev ]

Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Brijesh Singh <brijesh.singh@amd.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Borislav Petkov <bp@suse.de>
Tested-by: Borislav Petkov <bp@suse.de>
Cc: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: kvm@vger.kernel.org
Cc: David Laight <David.Laight@ACULAB.COM>
Cc: Borislav Petkov <bp@alien8.de>
Link: https://lkml.kernel.org/r/20171020143059.3291-14-brijesh.singh@amd.com

diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 11398d55aefaea3b03fe68bf65758e97c69513d2..93ae8aee178075da0110e026c546ab2db8674775 100644 (file)
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -266,6 +266,21 @@ static inline void slow_down_io(void)
 
 #endif
 
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+#include <linux/jump_label.h>
+
+extern struct static_key_false sev_enable_key;
+static inline bool sev_key_active(void)
+{
+       return static_branch_unlikely(&sev_enable_key);
+}
+
+#else /* !CONFIG_AMD_MEM_ENCRYPT */
+
+static inline bool sev_key_active(void) { return false; }
+
+#endif /* CONFIG_AMD_MEM_ENCRYPT */
+
 #define BUILDIO(bwl, bw, type)                                         \
 static inline void out##bwl(unsigned type value, int port)             \
 {                                                                      \
@@ -296,14 +311,34 @@ static inline unsigned type in##bwl##_p(int port)                 \
                                                                        \
 static inline void outs##bwl(int port, const void *addr, unsigned long count) \
 {                                                                      \
-       asm volatile("rep; outs" #bwl                                   \
-                    : "+S"(addr), "+c"(count) : "d"(port) : "memory"); \
+       if (sev_key_active()) {                                         \
+               unsigned type *value = (unsigned type *)addr;           \
+               while (count) {                                         \
+                       out##bwl(*value, port);                         \
+                       value++;                                        \
+                       count--;                                        \
+               }                                                       \
+       } else {                                                        \
+               asm volatile("rep; outs" #bwl                           \
+                            : "+S"(addr), "+c"(count)                  \
+                            : "d"(port) : "memory");                   \
+       }                                                               \
 }                                                                      \
                                                                        \
 static inline void ins##bwl(int port, void *addr, unsigned long count) \
 {                                                                      \
-       asm volatile("rep; ins" #bwl                                    \
-                    : "+D"(addr), "+c"(count) : "d"(port) : "memory"); \
+       if (sev_key_active()) {                                         \
+               unsigned type *value = (unsigned type *)addr;           \
+               while (count) {                                         \
+                       *value = in##bwl(port);                         \
+                       value++;                                        \
+                       count--;                                        \
+               }                                                       \
+       } else {                                                        \
+               asm volatile("rep; ins" #bwl                            \
+                            : "+D"(addr), "+c"(count)                  \
+                            : "d"(port) : "memory");                   \
+       }                                                               \
 }
 
 BUILDIO(b, b, char)

diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
index fd9c7bb516648418094b52e9871d12a961b583ab..d29b7831a0535ba9baf88e1d7fcd2b18bf6166b7 100644 (file)
--- a/arch/x86/mm/mem_encrypt.c
+++ b/arch/x86/mm/mem_encrypt.c
@@ -41,6 +41,8 @@ static char sme_cmdline_off[] __initdata = "off";
  */
 u64 sme_me_mask __section(.data) = 0;
 EXPORT_SYMBOL_GPL(sme_me_mask);
+DEFINE_STATIC_KEY_FALSE(sev_enable_key);
+EXPORT_SYMBOL_GPL(sev_enable_key);
 
 static bool sev_enabled __section(.data);
 
@@ -313,6 +315,12 @@ void __init mem_encrypt_init(void)
        if (sev_active())
                dma_ops = &sev_dma_ops;
 
+       /*
+        * With SEV, we need to unroll the rep string I/O instructions.
+        */
+       if (sev_active())
+               static_branch_enable(&sev_enable_key);
+
        pr_info("AMD %s active\n",
                sev_active() ? "Secure Encrypted Virtualization (SEV)"
                             : "Secure Memory Encryption (SME)");