i387: don't ever touch TS_USEDFPU directly, use helper functions

This creates three helper functions that do the TS_USEDFPU accesses, and
makes everybody that used to do it by hand use those helpers instead.

In addition, there's a couple of helper functions for the "change both
CR0.TS and TS_USEDFPU at the same time" case, and the places that do
that together have been changed to use those.  That means that we have
fewer random places that open-code this situation.

The intent is partly to clarify the code without actually changing any
semantics yet (since we clearly still have some hard to reproduce bug in
this area), but also to make it much easier to use another approach
entirely to caching the CR0.TS bit for software accesses.

Right now we use a bit in the thread-info 'status' variable (this patch
does not change that), but we might want to make it a full field of its
own or even make it a per-cpu variable.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 1e12c2d087e40d526954ac38f9e61038f154fdcf..548b2c07ac9ab0b48a1b522e712f2c0fe5f9cbb4 100644 (file)
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -279,6 +279,47 @@ static inline int restore_fpu_checking(struct task_struct *tsk)
        return fpu_restore_checking(&tsk->thread.fpu);
 }
 
+/*
+ * Software FPU state helpers. Careful: these need to
+ * be preemption protection *and* they need to be
+ * properly paired with the CR0.TS changes!
+ */
+static inline int __thread_has_fpu(struct thread_info *ti)
+{
+       return ti->status & TS_USEDFPU;
+}
+
+/* Must be paired with an 'stts' after! */
+static inline void __thread_clear_has_fpu(struct thread_info *ti)
+{
+       ti->status &= ~TS_USEDFPU;
+}
+
+/* Must be paired with a 'clts' before! */
+static inline void __thread_set_has_fpu(struct thread_info *ti)
+{
+       ti->status |= TS_USEDFPU;
+}
+
+/*
+ * Encapsulate the CR0.TS handling together with the
+ * software flag.
+ *
+ * These generally need preemption protection to work,
+ * do try to avoid using these on their own.
+ */
+static inline void __thread_fpu_end(struct thread_info *ti)
+{
+       __thread_clear_has_fpu(ti);
+       stts();
+}
+
+static inline void __thread_fpu_begin(struct thread_info *ti)
+{
+       clts();
+       __thread_set_has_fpu(ti);
+}
+
 /*
  * Signal frame handlers...
  */
@@ -287,23 +328,21 @@ extern int restore_i387_xstate(void __user *buf);
 
 static inline void __unlazy_fpu(struct task_struct *tsk)
 {
-       if (task_thread_info(tsk)->status & TS_USEDFPU) {
+       if (__thread_has_fpu(task_thread_info(tsk))) {
                __save_init_fpu(tsk);
-               task_thread_info(tsk)->status &= ~TS_USEDFPU;
-               stts();
+               __thread_fpu_end(task_thread_info(tsk));
        } else
                tsk->fpu_counter = 0;
 }
 
 static inline void __clear_fpu(struct task_struct *tsk)
 {
-       if (task_thread_info(tsk)->status & TS_USEDFPU) {
+       if (__thread_has_fpu(task_thread_info(tsk))) {
                /* Ignore delayed exceptions from user space */
                asm volatile("1: fwait\n"
                             "2:\n"
                             _ASM_EXTABLE(1b, 2b));
-               task_thread_info(tsk)->status &= ~TS_USEDFPU;
-               stts();
+               __thread_fpu_end(task_thread_info(tsk));
        }
 }
 
@@ -311,14 +350,14 @@ static inline void __clear_fpu(struct task_struct *tsk)
  * Were we in an interrupt that interrupted kernel mode?
  *
  * We can do a kernel_fpu_begin/end() pair *ONLY* if that
- * pair does nothing at all: TS_USEDFPU must be clear (so
+ * pair does nothing at all: the thread must not have fpu (so
  * that we don't try to save the FPU state), and TS must
  * be set (so that the clts/stts pair does nothing that is
  * visible in the interrupted kernel thread).
  */
 static inline bool interrupted_kernel_fpu_idle(void)
 {
-       return !(current_thread_info()->status & TS_USEDFPU) &&
+       return !__thread_has_fpu(current_thread_info()) &&
                (read_cr0() & X86_CR0_TS);
 }
 
@@ -356,9 +395,9 @@ static inline void kernel_fpu_begin(void)
 
        WARN_ON_ONCE(!irq_fpu_usable());
        preempt_disable();
-       if (me->status & TS_USEDFPU) {
+       if (__thread_has_fpu(me)) {
                __save_init_fpu(me->task);
-               me->status &= ~TS_USEDFPU;
+               __thread_clear_has_fpu(me);
                /* We do 'stts()' in kernel_fpu_end() */
        } else
                clts();
@@ -422,24 +461,21 @@ static inline void irq_ts_restore(int TS_state)
  */
 static inline int user_has_fpu(void)
 {
-       return current_thread_info()->status & TS_USEDFPU;
+       return __thread_has_fpu(current_thread_info());
 }
 
 static inline void user_fpu_end(void)
 {
        preempt_disable();
-       current_thread_info()->status &= ~TS_USEDFPU;
-       stts();
+       __thread_fpu_end(current_thread_info());
        preempt_enable();
 }
 
 static inline void user_fpu_begin(void)
 {
        preempt_disable();
-       if (!user_has_fpu()) {
-               clts();
-               current_thread_info()->status |= TS_USEDFPU;
-       }
+       if (!user_has_fpu())
+               __thread_fpu_begin(current_thread_info());
        preempt_enable();
 }
 
@@ -448,11 +484,10 @@ static inline void user_fpu_begin(void)
  */
 static inline void save_init_fpu(struct task_struct *tsk)
 {
-       WARN_ON_ONCE(!(task_thread_info(tsk)->status & TS_USEDFPU));
+       WARN_ON_ONCE(!__thread_has_fpu(task_thread_info(tsk)));
        preempt_disable();
        __save_init_fpu(tsk);
-       task_thread_info(tsk)->status &= ~TS_USEDFPU;
-       stts();
+       __thread_fpu_end(task_thread_info(tsk));
        preempt_enable();
 }
 

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 982433b5da30ec19aadd33978793ddcf2aac3e65..fc676e44c77f3b6a45fefacb0372c9fe846d0b29 100644 (file)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -588,7 +588,7 @@ void __math_state_restore(void)
                return;
        }
 
-       thread->status |= TS_USEDFPU;   /* So we fnsave on switch_to() */
+       __thread_set_has_fpu(thread);   /* clts in caller! */
        tsk->fpu_counter++;
 }
 

diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index 86f1f09a738abf565e2c793ad05ae677d49fb666..a0bcd0dbc95173731267e2093ffc9bddd61323a1 100644 (file)
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -47,7 +47,7 @@ void __sanitize_i387_state(struct task_struct *tsk)
        if (!fx)
                return;
 
-       BUG_ON(task_thread_info(tsk)->status & TS_USEDFPU);
+       BUG_ON(__thread_has_fpu(task_thread_info(tsk)));
 
        xstate_bv = tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv;
 

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index d29216c462b3ce56e33c3bf0ecf7595258b9705a..36091dd04b4b42db0529e2c117a006be2408db04 100644 (file)
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1457,7 +1457,7 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)
 #ifdef CONFIG_X86_64
        wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
 #endif
-       if (current_thread_info()->status & TS_USEDFPU)
+       if (__thread_has_fpu(current_thread_info()))
                clts();
        load_gdt(&__get_cpu_var(host_gdt));
 }