[PATCH] powerpc/8xx: Use 8MB D-TLB's for kernel static mapping faults

The following implements support for instantiation of 8MB D-TLB
entries for the kernel direct virtual mapping on 8xx, thus reducing TLB
space consumed for the kernel.

Test used: writing 40MB from /dev/zero to file in ext2fs over
RAMDISK.

$ time dd if=/dev/zero of=file bs=4k count=10000

VANILLA			8MB kernel data pages

real    0m11.485s	real    0m11.267s
user    0m0.218s        user    0m0.250s
sys     0m8.939s	sys     0m9.108s

real    0m11.518s	real    0m10.978s
user    0m0.203s 	user    0m0.222s
sys     0m9.585s	sys     0m9.138s

real    0m11.554s	real    0m10.967s
user    0m0.228s    	user    0m0.222s
sys     0m9.497s	sys     0m9.127s

real    0m11.633s	real	0m11.286s
user    0m0.214s	user    0m0.196s
sys     0m9.529s	sys     0m9.134s

and averages for both:

real	11.54750	real 11.12450

Which is a 3.6% improvement in execution time. More improvement is
expected for loads with larger kernel data footprint (real workloads).

Signed-off-by: Marcelo Tosatti <marcelo.tosatti@cyclades.com>
Signed-off-by: Paul Mackerras <paulus@samba.org>

diff --git a/arch/ppc/kernel/head_8xx.S b/arch/ppc/kernel/head_8xx.S
index de0978742221ae6eb0ddfb7e6a60120c7c48c4fa..3e6ca7f5843ff43575b2f5e5fe834b60bf3f03dd 100644 (file)
--- a/arch/ppc/kernel/head_8xx.S
+++ b/arch/ppc/kernel/head_8xx.S
@@ -375,6 +375,8 @@ DataStoreTLBMiss:
        lis     r11, swapper_pg_dir@h
        ori     r11, r11, swapper_pg_dir@l
        rlwimi  r10, r11, 0, 2, 19
+       stw     r12, 16(r0)
+       b LoadLargeDTLB
 3:
        lwz     r11, 0(r10)     /* Get the level 1 entry */
        rlwinm. r10, r11,0,0,19 /* Extract page descriptor page address */
@@ -430,6 +432,81 @@ DataStoreTLBMiss:
 InstructionTLBError:
        b       InstructionAccess
 
+LoadLargeDTLB:
+       li      r12, 0
+       lwz     r11, 0(r10)     /* Get the level 1 entry */
+       rlwinm. r10, r11,0,0,19 /* Extract page descriptor page address */
+       beq     3f              /* If zero, don't try to find a pte */
+
+       /* We have a pte table, so load fetch the pte from the table.
+        */
+       ori     r11, r11, 1     /* Set valid bit in physical L2 page */
+       DO_8xx_CPU6(0x3b80, r3)
+       mtspr   SPRN_MD_TWC, r11        /* Load pte table base address */
+       mfspr   r10, SPRN_MD_TWC        /* ....and get the pte address */
+       lwz     r10, 0(r10)     /* Get the pte */
+
+       /* Insert the Guarded flag into the TWC from the Linux PTE.
+        * It is bit 27 of both the Linux PTE and the TWC (at least
+        * I got that right :-).  It will be better when we can put
+        * this into the Linux pgd/pmd and load it in the operation
+        * above.
+        */
+       rlwimi  r11, r10, 0, 27, 27
+
+       rlwimi  r12, r10, 0, 0, 9       /* extract phys. addr */
+       mfspr   r3, SPRN_MD_EPN
+       rlwinm  r3, r3, 0, 0, 9         /* extract virtual address */
+       tophys(r3, r3)
+       cmpw    r3, r12                 /* only use 8M page if it is a direct 
+                                          kernel mapping */
+       bne     1f
+       ori     r11, r11, MD_PS8MEG
+       li      r12, 1
+       b       2f
+1:
+       li      r12, 0          /* can't use 8MB TLB, so zero r12. */
+2:
+       DO_8xx_CPU6(0x3b80, r3)
+       mtspr   SPRN_MD_TWC, r11
+
+       /* The Linux PTE won't go exactly into the MMU TLB.
+        * Software indicator bits 21, 22 and 28 must be clear.
+        * Software indicator bits 24, 25, 26, and 27 must be
+        * set.  All other Linux PTE bits control the behavior
+        * of the MMU.
+        */
+3:     li      r11, 0x00f0
+       rlwimi  r10, r11, 0, 24, 28     /* Set 24-27, clear 28 */
+       cmpwi   r12, 1
+       bne 4f
+       ori     r10, r10, 0x8
+
+       mfspr   r12, SPRN_MD_EPN
+       lis     r3, 0xff80              /* 10-19 must be clear for 8MB TLB */
+       ori     r3, r3, 0x0fff
+       and     r12, r3, r12
+       DO_8xx_CPU6(0x3780, r3)
+       mtspr   SPRN_MD_EPN, r12
+
+       lis     r3, 0xff80              /* 10-19 must be clear for 8MB TLB */
+       ori     r3, r3, 0x0fff
+       and     r10, r3, r10
+4:
+       DO_8xx_CPU6(0x3d80, r3)
+       mtspr   SPRN_MD_RPN, r10        /* Update TLB entry */
+
+       mfspr   r10, SPRN_M_TW  /* Restore registers */
+       lwz     r11, 0(r0)
+       mtcr    r11
+       lwz     r11, 4(r0)
+
+       lwz     r12, 16(r0)
+#ifdef CONFIG_8xx_CPU6
+       lwz     r3, 8(r0)
+#endif
+       rfi
+
 /* This is the data TLB error on the MPC8xx.  This could be due to
  * many reasons, including a dirty update to a pte.  We can catch that
  * one here, but anything else is an error.  First, we track down the