Merge branch 'core/percpu' into stackprotector

Conflicts:
	arch/x86/include/asm/pda.h
	arch/x86/include/asm/system.h

Also, moved include/asm-x86/stackprotector.h to arch/x86/include/asm.

Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --cc arch/x86/Kconfig
Simple merge

diff --cc arch/x86/include/asm/pda.h
index 3fea2fdb330294b7dfc68603565b20654182bdfa,c31ca048a901f71c585bfce5032541731847ffa6..5976cd803e9a2609acfbb1ff36b9576fa4af98ec
--- 1/arch/x86/include/asm/pda.h
--- 2/arch/x86/include/asm/pda.h
+++ b/arch/x86/include/asm/pda.h
@@@ -9,114 -11,30 +11,28 @@@
  
  /* Per processor datastructure. %gs points to it while the kernel runs */
  struct x8664_pda {
-       struct task_struct *pcurrent;   /* 0  Current process */
-       unsigned long data_offset;      /* 8 Per cpu data offset from linker
-                                          address */
-       unsigned long kernelstack;      /* 16 top of kernel stack for current */
-       unsigned long oldrsp;           /* 24 user rsp for system call */
-       int irqcount;                   /* 32 Irq nesting counter. Starts -1 */
-       unsigned int cpunumber;         /* 36 Logical CPU number */
+       unsigned long unused1;
+       unsigned long unused2;
+       unsigned long unused3;
+       unsigned long unused4;
+       int unused5;
+       unsigned int unused6;           /* 36 was cpunumber */
 -#ifdef CONFIG_CC_STACKPROTECTOR
        unsigned long stack_canary;     /* 40 stack canary value */
                                        /* gcc-ABI: this canary MUST be at
                                           offset 40!!! */
-       char *irqstackptr;
-       short nodenumber;               /* number of current node (32k max) */
 -#endif
        short in_bootmem;               /* pda lives in bootmem */
-       unsigned int __softirq_pending;
-       unsigned int __nmi_count;       /* number of NMI on this CPUs */
-       short mmu_state;
-       short isidle;
-       struct mm_struct *active_mm;
-       unsigned apic_timer_irqs;
-       unsigned irq0_irqs;
-       unsigned irq_resched_count;
-       unsigned irq_call_count;
-       unsigned irq_tlb_count;
-       unsigned irq_thermal_count;
-       unsigned irq_threshold_count;
-       unsigned irq_spurious_count;
  } ____cacheline_aligned_in_smp;
  
- extern struct x8664_pda **_cpu_pda;
+ DECLARE_PER_CPU(struct x8664_pda, __pda);
  extern void pda_init(int);
  
- #define cpu_pda(i) (_cpu_pda[i])
+ #define cpu_pda(cpu)          (&per_cpu(__pda, cpu))
  
- /*
-  * There is no fast way to get the base address of the PDA, all the accesses
-  * have to mention %fs/%gs.  So it needs to be done this Torvaldian way.
-  */
- extern void __bad_pda_field(void) __attribute__((noreturn));
- 
- /*
-  * proxy_pda doesn't actually exist, but tell gcc it is accessed for
-  * all PDA accesses so it gets read/write dependencies right.
-  */
- extern struct x8664_pda _proxy_pda;
- 
- #define pda_offset(field) offsetof(struct x8664_pda, field)
- 
- #define pda_to_op(op, field, val)                                     \
- do {                                                                  \
-       typedef typeof(_proxy_pda.field) T__;                           \
-       if (0) { T__ tmp__; tmp__ = (val); }    /* type checking */     \
-       switch (sizeof(_proxy_pda.field)) {                             \
-       case 2:                                                         \
-               asm(op "w %1,%%gs:%c2" :                                \
-                   "+m" (_proxy_pda.field) :                           \
-                   "ri" ((T__)val),                                    \
-                   "i"(pda_offset(field)));                            \
-               break;                                                  \
-       case 4:                                                         \
-               asm(op "l %1,%%gs:%c2" :                                \
-                   "+m" (_proxy_pda.field) :                           \
-                   "ri" ((T__)val),                                    \
-                   "i" (pda_offset(field)));                           \
-               break;                                                  \
-       case 8:                                                         \
-               asm(op "q %1,%%gs:%c2":                                 \
-                   "+m" (_proxy_pda.field) :                           \
-                   "ri" ((T__)val),                                    \
-                   "i"(pda_offset(field)));                            \
-               break;                                                  \
-       default:                                                        \
-               __bad_pda_field();                                      \
-       }                                                               \
- } while (0)
- 
- #define pda_from_op(op, field)                        \
- ({                                            \
-       typeof(_proxy_pda.field) ret__;         \
-       switch (sizeof(_proxy_pda.field)) {     \
-       case 2:                                 \
-               asm(op "w %%gs:%c1,%0" :        \
-                   "=r" (ret__) :              \
-                   "i" (pda_offset(field)),    \
-                   "m" (_proxy_pda.field));    \
-               break;                          \
-       case 4:                                 \
-               asm(op "l %%gs:%c1,%0":         \
-                   "=r" (ret__):               \
-                   "i" (pda_offset(field)),    \
-                   "m" (_proxy_pda.field));    \
-               break;                          \
-       case 8:                                 \
-               asm(op "q %%gs:%c1,%0":         \
-                   "=r" (ret__) :              \
-                   "i" (pda_offset(field)),    \
-                   "m" (_proxy_pda.field));    \
-               break;                          \
-       default:                                \
-               __bad_pda_field();              \
-       }                                       \
-       ret__;                                  \
- })
- 
- #define read_pda(field)               pda_from_op("mov", field)
- #define write_pda(field, val) pda_to_op("mov", field, val)
- #define add_pda(field, val)   pda_to_op("add", field, val)
- #define sub_pda(field, val)   pda_to_op("sub", field, val)
- #define or_pda(field, val)    pda_to_op("or", field, val)
+ #define read_pda(field)               percpu_read(__pda.field)
+ #define write_pda(field, val) percpu_write(__pda.field, val)
+ #define add_pda(field, val)   percpu_add(__pda.field, val)
+ #define sub_pda(field, val)   percpu_sub(__pda.field, val)
+ #define or_pda(field, val)    percpu_or(__pda.field, val)
  
  /* This is not atomic against other CPUs -- CPU preemption needs to be off */
  #define test_and_clear_bit_pda(bit, field)                            \
@@@ -130,8 -42,4 +40,6 @@@
  
  #endif
  
- #define PDA_STACKOFFSET (5*8)
- 
 +#define refresh_stack_canary() write_pda(stack_canary, current->stack_canary)
 +
  #endif /* _ASM_X86_PDA_H */

diff --cc arch/x86/include/asm/stackprotector.h
index 0000000000000000000000000000000000000000,0000000000000000000000000000000000000000..c7f0d10bae7b5c8c6012d69836ee0f156d8149d2
new file mode 100644 (file)
--- /dev/null
--- /dev/null
+++ b/arch/x86/include/asm/stackprotector.h
@@@ -1,0 -1,0 +1,39 @@@
++#ifndef _ASM_STACKPROTECTOR_H
++#define _ASM_STACKPROTECTOR_H 1
++
++#include <asm/tsc.h>
++#include <asm/pda.h>
++
++/*
++ * Initialize the stackprotector canary value.
++ *
++ * NOTE: this must only be called from functions that never return,
++ * and it must always be inlined.
++ */
++static __always_inline void boot_init_stack_canary(void)
++{
++      u64 canary;
++      u64 tsc;
++
++      /*
++       * If we're the non-boot CPU, nothing set the PDA stack
++       * canary up for us - and if we are the boot CPU we have
++       * a 0 stack canary. This is a good place for updating
++       * it, as we wont ever return from this function (so the
++       * invalid canaries already on the stack wont ever
++       * trigger).
++       *
++       * We both use the random pool and the current TSC as a source
++       * of randomness. The TSC only matters for very early init,
++       * there it already has some randomness on most systems. Later
++       * on during the bootup the random pool has true entropy too.
++       */
++      get_random_bytes(&canary, sizeof(canary));
++      tsc = __native_read_tsc();
++      canary += tsc + (tsc << 32UL);
++
++      current->stack_canary = canary;
++      write_pda(stack_canary, canary);
++}
++
++#endif

diff --cc arch/x86/include/asm/system.h
index 2f6340a44291e9cd3300909d26b625697fc1a7d9,d1dc27dba36da4ed20a72c3014adaa901b7ea073..8cadfe9b1194ebfe78a4fb4bf2bce6c0e3a8dbba
--- 1/arch/x86/include/asm/system.h
--- 2/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@@ -94,9 -94,7 +94,9 @@@ do {                                                                  
             "call __switch_to\n\t"                                       \
             ".globl thread_return\n"                                     \
             "thread_return:\n\t"                                         \
-            "movq %%gs:%P[pda_pcurrent],%%rsi\n\t"                       \
+            "movq "__percpu_arg([current_task])",%%rsi\n\t"              \
 +           "movq %P[task_canary](%%rsi),%%r8\n\t"                       \
 +           "movq %%r8,%%gs:%P[pda_canary]\n\t"                          \
             "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
             LOCK_PREFIX "btr  %[tif_fork],%P[ti_flags](%%r8)\n\t"        \
             "movq %%rax,%%rdi\n\t"                                       \
@@@ -108,9 -106,7 +108,9 @@@
               [ti_flags] "i" (offsetof(struct thread_info, flags)),      \
               [tif_fork] "i" (TIF_FORK),                                 \
               [thread_info] "i" (offsetof(struct task_struct, stack)),   \
 -             [current_task] "m" (per_cpu_var(current_task))             \
 +             [task_canary] "i" (offsetof(struct task_struct, stack_canary)),\
-              [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent)), \
++             [current_task] "m" (per_cpu_var(current_task)),            \
 +             [pda_canary] "i" (offsetof(struct x8664_pda, stack_canary))\
             : "memory", "cc" __EXTRA_CLOBBER)
  #endif
  

diff --cc arch/x86/kernel/process_64.c
index efb0396e19bf4bc8f9d90b7c913558bf0c6e2977,4523ff88a69d47e880c0d046aa1fa12db950d109..aa89eabf09e05a40d5bdfe641a2250210b765126
--- 1/arch/x86/kernel/process_64.c
--- 2/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@@ -625,14 -619,15 +631,14 @@@ __switch_to(struct task_struct *prev_p
        /*
         * Switch the PDA and FPU contexts.
         */
-       prev->usersp = read_pda(oldrsp);
-       write_pda(oldrsp, next->usersp);
-       write_pda(pcurrent, next_p);
+       prev->usersp = percpu_read(old_rsp);
+       percpu_write(old_rsp, next->usersp);
+       percpu_write(current_task, next_p);
  
-       write_pda(kernelstack,
+       percpu_write(kernel_stack,
                  (unsigned long)task_stack_page(next_p) +
-                 THREAD_SIZE - PDA_STACKOFFSET);
+                 THREAD_SIZE - KERNEL_STACK_OFFSET);
  #ifdef CONFIG_CC_STACKPROTECTOR
 -      write_pda(stack_canary, next_p->stack_canary);
        /*
         * Build time only check to make sure the stack_canary is at
         * offset 40 in the pda; this is a gcc ABI requirement

diff --cc arch/x86/mm/fault.c
Simple merge

diff --cc include/linux/magic.h
Simple merge

diff --cc include/linux/sched.h
Simple merge

diff --cc init/main.c
index 07da4dea50c3f86b60beb3202450b1444b83dcc6,844209453c02c2248114972fca3f0fcddbdf513d..bfe4fb0c98428bdb4cfc9fb4b2a6cbb1084a0da1
--- 1/init/main.c
--- 2/init/main.c
+++ b/init/main.c
@@@ -561,15 -537,8 +538,14 @@@ asmlinkage void __init start_kernel(voi
         * Need to run as early as possible, to initialize the
         * lockdep hash:
         */
-       unwind_init();
        lockdep_init();
        debug_objects_early_init();
 +
 +      /*
 +       * Set up the the initial canary ASAP:
 +       */
 +      boot_init_stack_canary();
 +
        cgroup_init_early();
  
        local_irq_disable();

diff --cc kernel/exit.c
Simple merge

diff --cc kernel/fork.c
Simple merge

diff --cc kernel/panic.c
Simple merge

diff --cc kernel/sched.c
Simple merge