x64, x2apic/intr-remap: basic apic ops support

Introduce basic apic operations which handle the apic programming. This
will be used later to introduce another specific operations for x2apic.

For the perfomance critial accesses like IPI's, EOI etc, we use the
native operations as they are already referenced by different
indirections like genapic, irq_chip etc.

64bit Paravirt ops can also define their apic operations accordingly.

Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
Cc: akpm@linux-foundation.org
Cc: arjan@linux.intel.com
Cc: andi@firstfloor.org
Cc: ebiederm@xmission.com
Cc: jbarnes@virtuousgeek.org
Cc: steiner@sgi.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/kernel/apic_32.c b/arch/x86/kernel/apic_32.c
index 3e58b676d23b8fd96a87823e30fcc4566d60aad6..2a83c07bd88731e5d958a79facd59bde74b09143 100644 (file)
--- a/arch/x86/kernel/apic_32.c
+++ b/arch/x86/kernel/apic_32.c
@@ -145,6 +145,12 @@ static int modern_apic(void)
        return lapic_get_version() >= 0x14;
 }
 
+void apic_icr_write(u32 low, u32 id)
+{
+       apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(id));
+       apic_write_around(APIC_ICR, low);
+}
+
 void apic_wait_icr_idle(void)
 {
        while (apic_read(APIC_ICR) & APIC_ICR_BUSY)

diff --git a/arch/x86/kernel/apic_64.c b/arch/x86/kernel/apic_64.c
index 3963f590c3d4d6c10a7b0e6e53e9416e6426322b..9bb040689b31aa0d8b6850b26d93d0905f7ff8cd 100644 (file)
--- a/arch/x86/kernel/apic_64.c
+++ b/arch/x86/kernel/apic_64.c
@@ -119,13 +119,13 @@ static int modern_apic(void)
        return lapic_get_version() >= 0x14;
 }
 
-void apic_wait_icr_idle(void)
+void xapic_wait_icr_idle(void)
 {
        while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
                cpu_relax();
 }
 
-u32 safe_apic_wait_icr_idle(void)
+u32 safe_xapic_wait_icr_idle(void)
 {
        u32 send_status;
        int timeout;
@@ -141,6 +141,36 @@ u32 safe_apic_wait_icr_idle(void)
        return send_status;
 }
 
+void xapic_icr_write(u32 low, u32 id)
+{
+       apic_write(APIC_ICR2, id << 24);
+       apic_write(APIC_ICR, low);
+}
+
+u64 xapic_icr_read(void)
+{
+       u32 icr1, icr2;
+
+       icr2 = apic_read(APIC_ICR2);
+       icr1 = apic_read(APIC_ICR);
+
+       return (icr1 | ((u64)icr2 << 32));
+}
+
+static struct apic_ops xapic_ops = {
+       .read = native_apic_mem_read,
+       .write = native_apic_mem_write,
+       .write_atomic = native_apic_mem_write_atomic,
+       .icr_read = xapic_icr_read,
+       .icr_write = xapic_icr_write,
+       .wait_icr_idle = xapic_wait_icr_idle,
+       .safe_wait_icr_idle = safe_xapic_wait_icr_idle,
+};
+
+struct apic_ops __read_mostly *apic_ops = &xapic_ops;
+
+EXPORT_SYMBOL_GPL(apic_ops);
+
 /**
  * enable_NMI_through_LVT0 - enable NMI through local vector table 0
  */

diff --git a/arch/x86/kernel/io_apic_64.c b/arch/x86/kernel/io_apic_64.c
index 84dd63c13d6300c841748361a7935db15849f252..b62d42ef928393ea7cc5d4ab40b6adbc10048a28 100644 (file)
--- a/arch/x86/kernel/io_apic_64.c
+++ b/arch/x86/kernel/io_apic_64.c
@@ -1157,6 +1157,7 @@ static __apicdebuginit void print_APIC_bitfield (int base)
 void __apicdebuginit print_local_APIC(void * dummy)
 {
        unsigned int v, ver, maxlvt;
+       unsigned long icr;
 
        if (apic_verbosity == APIC_QUIET)
                return;
@@ -1200,10 +1201,9 @@ void __apicdebuginit print_local_APIC(void * dummy)
        v = apic_read(APIC_ESR);
        printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
 
-       v = apic_read(APIC_ICR);
-       printk(KERN_DEBUG "... APIC ICR: %08x\n", v);
-       v = apic_read(APIC_ICR2);
-       printk(KERN_DEBUG "... APIC ICR2: %08x\n", v);
+       icr = apic_icr_read();
+       printk(KERN_DEBUG "... APIC ICR: %08x\n", icr);
+       printk(KERN_DEBUG "... APIC ICR2: %08x\n", icr >> 32);
 
        v = apic_read(APIC_LVTT);
        printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);

diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index e0f571d58c19c0bfa4eeee39d89972ed2239f55f..b80105a0f47402a47481f6e893f9e5b0604d6ded 100644 (file)
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -360,9 +360,11 @@ struct pv_cpu_ops pv_cpu_ops = {
 
 struct pv_apic_ops pv_apic_ops = {
 #ifdef CONFIG_X86_LOCAL_APIC
-       .apic_write = native_apic_write,
-       .apic_write_atomic = native_apic_write_atomic,
-       .apic_read = native_apic_read,
+#ifnded CONFIG_X86_64
+       .apic_write = native_apic_mem_write,
+       .apic_write_atomic = native_apic_mem_write_atomic,
+       .apic_read = native_apic_mem_read,
+#endif
        .setup_boot_clock = setup_boot_APIC_clock,
        .setup_secondary_clock = setup_secondary_APIC_clock,
        .startup_ipi_hook = paravirt_nop,

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index f35c2d8016ac412c1c0433bf4382829c1d2cef31..c55263b3df02633f40f33f6b18653848a15b9207 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -123,7 +123,6 @@ EXPORT_PER_CPU_SYMBOL(cpu_info);
 
 static atomic_t init_deasserted;
 
-static int boot_cpu_logical_apicid;
 
 /* representing cpus for which sibling maps can be computed */
 static cpumask_t cpu_sibling_setup_map;
@@ -165,6 +164,8 @@ static void unmap_cpu_to_node(int cpu)
 #endif
 
 #ifdef CONFIG_X86_32
+static int boot_cpu_logical_apicid;
+
 u8 cpu_2_logical_apicid[NR_CPUS] __read_mostly =
                                        { [0 ... NR_CPUS-1] = BAD_APICID };
 
@@ -546,8 +547,7 @@ static inline void __inquire_remote_apic(int apicid)
                        printk(KERN_CONT
                               "a previous APIC delivery may have failed\n");
 
-               apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
-               apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);
+               apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
 
                timeout = 0;
                do {
@@ -579,11 +579,9 @@ wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
        int maxlvt;
 
        /* Target chip */
-       apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(logical_apicid));
-
        /* Boot on the stack */
        /* Kick the second */
-       apic_write_around(APIC_ICR, APIC_DM_NMI | APIC_DEST_LOGICAL);
+       apic_icr_write(APIC_DM_NMI | APIC_DEST_LOGICAL, logical_apicid);
 
        Dprintk("Waiting for send to finish...\n");
        send_status = safe_apic_wait_icr_idle();
@@ -639,13 +637,11 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
        /*
         * Turn INIT on target chip
         */
-       apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
-
        /*
         * Send IPI
         */
-       apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
-                               | APIC_DM_INIT);
+       apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
+                      phys_apicid);
 
        Dprintk("Waiting for send to finish...\n");
        send_status = safe_apic_wait_icr_idle();
@@ -655,10 +651,8 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
        Dprintk("Deasserting INIT.\n");
 
        /* Target chip */
-       apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
-
        /* Send IPI */
-       apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
+       apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
 
        Dprintk("Waiting for send to finish...\n");
        send_status = safe_apic_wait_icr_idle();
@@ -703,12 +697,10 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
                 */
 
                /* Target chip */
-               apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
-
                /* Boot on the stack */
                /* Kick the second */
-               apic_write_around(APIC_ICR, APIC_DM_STARTUP
-                                       | (start_eip >> 12));
+               apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
+                              phys_apicid);
 
                /*
                 * Give the other CPU some time to accept the IPI.
@@ -1147,7 +1139,9 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
         * Setup boot CPU information
         */
        smp_store_cpu_info(0); /* Final full version of the data */
+#ifdef CONFIG_X86_32
        boot_cpu_logical_apicid = logical_smp_processor_id();
+#endif
        current_thread_info()->cpu = 0;  /* needed? */
        set_cpu_sibling_map(0);
 

diff --git a/include/asm-x86/apic.h b/include/asm-x86/apic.h
index 4e2c1e517f0652fb1877c378f760fdb03fe6e8bb..6fda195337c53a92303aaa3450eaafa2feba9b64 100644 (file)
--- a/include/asm-x86/apic.h
+++ b/include/asm-x86/apic.h
@@ -47,32 +47,59 @@ extern int disable_apic;
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
 #else
-#define apic_write native_apic_write
-#define apic_write_atomic native_apic_write_atomic
-#define apic_read native_apic_read
+#ifndef CONFIG_X86_64
+#define apic_write native_apic_mem_write
+#define apic_write_atomic native_apic_mem_write_atomic
+#define apic_read native_apic_mem_read
+#endif
 #define setup_boot_clock setup_boot_APIC_clock
 #define setup_secondary_clock setup_secondary_APIC_clock
 #endif
 
 extern int is_vsmp_box(void);
 
-static inline void native_apic_write(unsigned long reg, u32 v)
+static inline void native_apic_mem_write(u32 reg, u32 v)
 {
        *((volatile u32 *)(APIC_BASE + reg)) = v;
 }
 
-static inline void native_apic_write_atomic(unsigned long reg, u32 v)
+static inline void native_apic_mem_write_atomic(u32 reg, u32 v)
 {
        (void)xchg((u32 *)(APIC_BASE + reg), v);
 }
 
-static inline u32 native_apic_read(unsigned long reg)
+static inline u32 native_apic_mem_read(u32 reg)
 {
        return *((volatile u32 *)(APIC_BASE + reg));
 }
 
+#ifdef CONFIG_X86_32
 extern void apic_wait_icr_idle(void);
 extern u32 safe_apic_wait_icr_idle(void);
+extern void apic_icr_write(u32 low, u32 id);
+#else
+
+struct apic_ops {
+       u32 (*read)(u32 reg);
+       void (*write)(u32 reg, u32 v);
+       void (*write_atomic)(u32 reg, u32 v);
+       u64 (*icr_read)(void);
+       void (*icr_write)(u32 low, u32 high);
+       void (*wait_icr_idle)(void);
+       u32 (*safe_wait_icr_idle)(void);
+};
+
+extern struct apic_ops *apic_ops;
+
+#define apic_read (apic_ops->read)
+#define apic_write (apic_ops->write)
+#define apic_write_atomic (apic_ops->write_atomic)
+#define apic_icr_read (apic_ops->icr_read)
+#define apic_icr_write (apic_ops->icr_write)
+#define apic_wait_icr_idle (apic_ops->wait_icr_idle)
+#define safe_apic_wait_icr_idle (apic_ops->safe_wait_icr_idle)
+#endif
+
 extern int get_physical_broadcast(void);
 
 #ifdef CONFIG_X86_GOOD_APIC
@@ -95,7 +122,11 @@ static inline void ack_APIC_irq(void)
         */
 
        /* Docs say use 0 for future compatibility */
+#ifdef CONFIG_X86_32
        apic_write_around(APIC_EOI, 0);
+#else
+       native_apic_mem_write(APIC_EOI, 0);
+#endif
 }
 
 extern int lapic_get_maxlvt(void);

diff --git a/include/asm-x86/ipi.h b/include/asm-x86/ipi.h
index 196d63c28aa44aa3365977da4b729d59444fcb69..3d8d6a6c1f8e5f8435db4e6be54b0608d0872ace 100644 (file)
--- a/include/asm-x86/ipi.h
+++ b/include/asm-x86/ipi.h
@@ -49,6 +49,12 @@ static inline int __prepare_ICR2(unsigned int mask)
        return SET_APIC_DEST_FIELD(mask);
 }
 
+static inline void __xapic_wait_icr_idle(void)
+{
+       while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY)
+               cpu_relax();
+}
+
 static inline void __send_IPI_shortcut(unsigned int shortcut, int vector,
                                       unsigned int dest)
 {
@@ -64,7 +70,7 @@ static inline void __send_IPI_shortcut(unsigned int shortcut, int vector,
        /*
         * Wait for idle.
         */
-       apic_wait_icr_idle();
+       __xapic_wait_icr_idle();
 
        /*
         * No need to touch the target chip field
@@ -74,7 +80,7 @@ static inline void __send_IPI_shortcut(unsigned int shortcut, int vector,
        /*
         * Send the IPI. The write to APIC_ICR fires this off.
         */
-       apic_write(APIC_ICR, cfg);
+       native_apic_mem_write(APIC_ICR, cfg);
 }
 
 /*
@@ -92,13 +98,13 @@ static inline void __send_IPI_dest_field(unsigned int mask, int vector,
        if (unlikely(vector == NMI_VECTOR))
                safe_apic_wait_icr_idle();
        else
-               apic_wait_icr_idle();
+               __xapic_wait_icr_idle();
 
        /*
         * prepare target chip field
         */
        cfg = __prepare_ICR2(mask);
-       apic_write(APIC_ICR2, cfg);
+       native_apic_mem_write(APIC_ICR2, cfg);
 
        /*
         * program the ICR
@@ -108,7 +114,7 @@ static inline void __send_IPI_dest_field(unsigned int mask, int vector,
        /*
         * Send the IPI. The write to APIC_ICR fires this off.
         */
-       apic_write(APIC_ICR, cfg);
+       native_apic_mem_write(APIC_ICR, cfg);
 }
 
 static inline void send_IPI_mask_sequence(cpumask_t mask, int vector)

diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h
index ef5e8ec6a6ab7431f0c799a2b56a4cd5f18fe5e9..10adac02e6db3286a9bd240d2e770283709f388a 100644 (file)
--- a/include/asm-x86/paravirt.h
+++ b/include/asm-x86/paravirt.h
@@ -891,6 +891,7 @@ static inline void slow_down_io(void)
 /*
  * Basic functions accessing APICs.
  */
+#ifndef CONFIG_X86_64
 static inline void apic_write(unsigned long reg, u32 v)
 {
        PVOP_VCALL2(pv_apic_ops.apic_write, reg, v);
@@ -905,6 +906,7 @@ static inline u32 apic_read(unsigned long reg)
 {
        return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg);
 }
+#endif
 
 static inline void setup_boot_clock(void)
 {

diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 9848715fbd9ef7b448d6089b64f2dd8d3bd47c3d..d9d007d227855a0c3fee1bb8f014d4bd0df22fd6 100644 (file)
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -158,13 +158,13 @@ extern int safe_smp_processor_id(void);
 
 #ifdef CONFIG_X86_LOCAL_APIC
 
+#ifndef CONFIG_X86_64
 static inline int logical_smp_processor_id(void)
 {
        /* we don't want to mark this access volatile - bad code generation */
        return GET_APIC_LOGICAL_ID(*(u32 *)(APIC_BASE + APIC_LDR));
 }
 
-#ifndef CONFIG_X86_64
 static inline unsigned int read_apic_id(void)
 {
        return *(u32 *)(APIC_BASE + APIC_ID);