MIPS: Transparent Huge Pages support

Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index b47d591c03ddedabfeaaf45998a63bb63fad5198..9934a4687ac4ffa74745759af03936f57b16f6d8 100644 (file)
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -19,6 +19,7 @@ config MIPS
        select HAVE_KRETPROBES
        select HAVE_DEBUG_KMEMLEAK
        select ARCH_BINFMT_ELF_RANDOMIZE_PIE
+       select HAVE_ARCH_TRANSPARENT_HUGEPAGE
        select RTC_LIB if !MACH_LOONGSON
        select GENERIC_ATOMIC64 if !64BIT
        select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
@@ -1372,6 +1373,7 @@ config CPU_R4X00
        depends on SYS_HAS_CPU_R4X00
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
+       select CPU_SUPPORTS_HUGEPAGES
        help
          MIPS Technologies R4000-series processors other than 4300, including
          the R4000, R4400, R4600, and 4700.
@@ -1382,12 +1384,14 @@ config CPU_TX49XX
        select CPU_HAS_PREFETCH
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
+       select CPU_SUPPORTS_HUGEPAGES
 
 config CPU_R5000
        bool "R5000"
        depends on SYS_HAS_CPU_R5000
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
+       select CPU_SUPPORTS_HUGEPAGES
        help
          MIPS Technologies R5000-series processors other than the Nevada.
 
@@ -1396,6 +1400,7 @@ config CPU_R5432
        depends on SYS_HAS_CPU_R5432
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
+       select CPU_SUPPORTS_HUGEPAGES
 
 config CPU_R5500
        bool "R5500"
@@ -1421,6 +1426,7 @@ config CPU_NEVADA
        depends on SYS_HAS_CPU_NEVADA
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
+       select CPU_SUPPORTS_HUGEPAGES
        help
          QED / PMC-Sierra RM52xx-series ("Nevada") processors.
 
@@ -1441,6 +1447,7 @@ config CPU_R10000
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
        select CPU_SUPPORTS_HIGHMEM
+       select CPU_SUPPORTS_HUGEPAGES
        help
          MIPS Technologies R10000-series processors.
 
@@ -1451,6 +1458,7 @@ config CPU_RM7000
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
        select CPU_SUPPORTS_HIGHMEM
+       select CPU_SUPPORTS_HUGEPAGES
 
 config CPU_RM9000
        bool "RM9000"
@@ -1459,6 +1467,7 @@ config CPU_RM9000
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
        select CPU_SUPPORTS_HIGHMEM
+       select CPU_SUPPORTS_HUGEPAGES
        select WEAK_ORDERING
 
 config CPU_SB1
@@ -1467,6 +1476,7 @@ config CPU_SB1
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
        select CPU_SUPPORTS_HIGHMEM
+       select CPU_SUPPORTS_HUGEPAGES
        select WEAK_ORDERING
 
 config CPU_CAVIUM_OCTEON
@@ -1530,9 +1540,9 @@ config CPU_XLR
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
        select CPU_SUPPORTS_HIGHMEM
+       select CPU_SUPPORTS_HUGEPAGES
        select WEAK_ORDERING
        select WEAK_REORDERING_BEYOND_LLSC
-       select CPU_SUPPORTS_HUGEPAGES
        help
          Netlogic Microsystems XLR/XLS processors.
 
@@ -1593,6 +1603,7 @@ config CPU_LOONGSON2
        select CPU_SUPPORTS_32BIT_KERNEL
        select CPU_SUPPORTS_64BIT_KERNEL
        select CPU_SUPPORTS_HIGHMEM
+       select CPU_SUPPORTS_HUGEPAGES
 
 config CPU_LOONGSON1
        bool

diff --git a/arch/mips/include/asm/pgtable-64.h b/arch/mips/include/asm/pgtable-64.h
index f5b521d5a67d744e3f250c5a99e5023c77deaee0..c63191055e695c5a49812f3f32ed0c913d941a0a 100644 (file)
--- a/arch/mips/include/asm/pgtable-64.h
+++ b/arch/mips/include/asm/pgtable-64.h
@@ -175,7 +175,7 @@ static inline int pmd_none(pmd_t pmd)
 
 static inline int pmd_bad(pmd_t pmd)
 {
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
        /* pmd_huge(pmd) but inline */
        if (unlikely(pmd_val(pmd) & _PAGE_HUGE))
                return 0;

diff --git a/arch/mips/include/asm/pgtable-bits.h b/arch/mips/include/asm/pgtable-bits.h
index 1e2642c360a8fdc15f47cfb4add345d292533992..9ce1ac782448d92b478b0ea9d0cdca5f450bd8f8 100644 (file)
--- a/arch/mips/include/asm/pgtable-bits.h
+++ b/arch/mips/include/asm/pgtable-bits.h
@@ -137,8 +137,17 @@
 #define _PAGE_HUGE             ({BUG(); 1; })  /* Dummy value */
 #endif
 
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
+/* huge tlb page */
+#define _PAGE_SPLITTING_SHIFT  (_PAGE_HUGE_SHIFT + 1)
+#define _PAGE_SPLITTING                (1 << _PAGE_SPLITTING_SHIFT)
+#else
+#define _PAGE_SPLITTING_SHIFT  (_PAGE_HUGE_SHIFT)
+#define _PAGE_SPLITTING                ({BUG(); 1; })  /* Dummy value */
+#endif
+
 /* Page cannot be executed */
-#define _PAGE_NO_EXEC_SHIFT    (cpu_has_rixi ? _PAGE_HUGE_SHIFT + 1 : _PAGE_HUGE_SHIFT)
+#define _PAGE_NO_EXEC_SHIFT    (cpu_has_rixi ? _PAGE_SPLITTING_SHIFT + 1 : _PAGE_SPLITTING_SHIFT)
 #define _PAGE_NO_EXEC          ({BUG_ON(!cpu_has_rixi); 1 << _PAGE_NO_EXEC_SHIFT; })
 
 /* Page cannot be read */

diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h
index bc1c0fbf8d998ddf9f0358e041357f6c7aab306b..252202d24a84f50c467f11dff8937863150aedab 100644 (file)
--- a/arch/mips/include/asm/pgtable.h
+++ b/arch/mips/include/asm/pgtable.h
@@ -8,6 +8,7 @@
 #ifndef _ASM_PGTABLE_H
 #define _ASM_PGTABLE_H
 
+#include <linux/mmzone.h>
 #ifdef CONFIG_32BIT
 #include <asm/pgtable-32.h>
 #endif
@@ -94,7 +95,12 @@ extern void paging_init(void);
  * and a page entry and page directory to the page they refer to.
  */
 #define pmd_phys(pmd)          virt_to_phys((void *)pmd_val(pmd))
-#define pmd_page(pmd)          (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
+
+#define __pmd_page(pmd)                (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
+#ifndef CONFIG_TRANSPARENT_HUGEPAGE
+#define pmd_page(pmd)          __pmd_page(pmd)
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE  */
+
 #define pmd_page_vaddr(pmd)    pmd_val(pmd)
 
 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
@@ -374,6 +380,14 @@ static inline void update_mmu_cache(struct vm_area_struct *vma,
        __update_cache(vma, address, pte);
 }
 
+static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
+       unsigned long address, pmd_t *pmdp)
+{
+       pte_t pte = *(pte_t *)pmdp;
+
+       __update_tlb(vma, address, pte);
+}
+
 #define kern_addr_valid(addr)  (1)
 
 #ifdef CONFIG_64BIT_PHYS_ADDR
@@ -393,6 +407,157 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma,
                remap_pfn_range(vma, vaddr, pfn, size, prot)
 #endif
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+
+extern int has_transparent_hugepage(void);
+
+static inline int pmd_trans_huge(pmd_t pmd)
+{
+       return !!(pmd_val(pmd) & _PAGE_HUGE);
+}
+
+static inline pmd_t pmd_mkhuge(pmd_t pmd)
+{
+       pmd_val(pmd) |= _PAGE_HUGE;
+
+       return pmd;
+}
+
+static inline int pmd_trans_splitting(pmd_t pmd)
+{
+       return !!(pmd_val(pmd) & _PAGE_SPLITTING);
+}
+
+static inline pmd_t pmd_mksplitting(pmd_t pmd)
+{
+       pmd_val(pmd) |= _PAGE_SPLITTING;
+
+       return pmd;
+}
+
+extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+                      pmd_t *pmdp, pmd_t pmd);
+
+#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
+/* Extern to avoid header file madness */
+extern void pmdp_splitting_flush(struct vm_area_struct *vma,
+                                       unsigned long address,
+                                       pmd_t *pmdp);
+
+#define __HAVE_ARCH_PMD_WRITE
+static inline int pmd_write(pmd_t pmd)
+{
+       return !!(pmd_val(pmd) & _PAGE_WRITE);
+}
+
+static inline pmd_t pmd_wrprotect(pmd_t pmd)
+{
+       pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_SILENT_WRITE);
+       return pmd;
+}
+
+static inline pmd_t pmd_mkwrite(pmd_t pmd)
+{
+       pmd_val(pmd) |= _PAGE_WRITE;
+       if (pmd_val(pmd) & _PAGE_MODIFIED)
+               pmd_val(pmd) |= _PAGE_SILENT_WRITE;
+
+       return pmd;
+}
+
+static inline int pmd_dirty(pmd_t pmd)
+{
+       return !!(pmd_val(pmd) & _PAGE_MODIFIED);
+}
+
+static inline pmd_t pmd_mkclean(pmd_t pmd)
+{
+       pmd_val(pmd) &= ~(_PAGE_MODIFIED | _PAGE_SILENT_WRITE);
+       return pmd;
+}
+
+static inline pmd_t pmd_mkdirty(pmd_t pmd)
+{
+       pmd_val(pmd) |= _PAGE_MODIFIED;
+       if (pmd_val(pmd) & _PAGE_WRITE)
+               pmd_val(pmd) |= _PAGE_SILENT_WRITE;
+
+       return pmd;
+}
+
+static inline int pmd_young(pmd_t pmd)
+{
+       return !!(pmd_val(pmd) & _PAGE_ACCESSED);
+}
+
+static inline pmd_t pmd_mkold(pmd_t pmd)
+{
+       pmd_val(pmd) &= ~(_PAGE_ACCESSED|_PAGE_SILENT_READ);
+
+       return pmd;
+}
+
+static inline pmd_t pmd_mkyoung(pmd_t pmd)
+{
+       pmd_val(pmd) |= _PAGE_ACCESSED;
+
+       if (cpu_has_rixi) {
+               if (!(pmd_val(pmd) & _PAGE_NO_READ))
+                       pmd_val(pmd) |= _PAGE_SILENT_READ;
+       } else {
+               if (pmd_val(pmd) & _PAGE_READ)
+                       pmd_val(pmd) |= _PAGE_SILENT_READ;
+       }
+
+       return pmd;
+}
+
+/* Extern to avoid header file madness */
+extern pmd_t mk_pmd(struct page *page, pgprot_t prot);
+
+static inline unsigned long pmd_pfn(pmd_t pmd)
+{
+       return pmd_val(pmd) >> _PFN_SHIFT;
+}
+
+static inline struct page *pmd_page(pmd_t pmd)
+{
+       if (pmd_trans_huge(pmd))
+               return pfn_to_page(pmd_pfn(pmd));
+
+       return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT);
+}
+
+static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+{
+       pmd_val(pmd) = (pmd_val(pmd) & _PAGE_CHG_MASK) | pgprot_val(newprot);
+       return pmd;
+}
+
+static inline pmd_t pmd_mknotpresent(pmd_t pmd)
+{
+       pmd_val(pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY);
+
+       return pmd;
+}
+
+/*
+ * The generic version pmdp_get_and_clear uses a version of pmd_clear() with a
+ * different prototype.
+ */
+#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
+static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
+                                      unsigned long address, pmd_t *pmdp)
+{
+       pmd_t old = *pmdp;
+
+       pmd_clear(pmdp);
+
+       return old;
+}
+
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
 #include <asm-generic/pgtable.h>
 
 /*

diff --git a/arch/mips/kernel/mips_ksyms.c b/arch/mips/kernel/mips_ksyms.c
index 3fc1691110dc52f82e8ec1271054c1411c122658..1ba8933683aaead539bf5beefb2f273f150e1ba8 100644 (file)
--- a/arch/mips/kernel/mips_ksyms.c
+++ b/arch/mips/kernel/mips_ksyms.c
@@ -11,7 +11,7 @@
 #include <linux/interrupt.h>
 #include <linux/export.h>
 #include <asm/checksum.h>
-#include <asm/pgtable.h>
+#include <linux/mm.h>
 #include <asm/uaccess.h>
 #include <asm/ftrace.h>
 

diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c
index 25407794edb4ff875bba1acb9ac5c4fa632ae78c..ee331bbd8f8aa13c263b8aa31c6bfdbab75e1101 100644 (file)
--- a/arch/mips/mm/pgtable-64.c
+++ b/arch/mips/mm/pgtable-64.c
@@ -11,6 +11,7 @@
 #include <asm/fixmap.h>
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
+#include <asm/tlbflush.h>
 
 void pgd_init(unsigned long page)
 {
@@ -61,6 +62,36 @@ void pmd_init(unsigned long addr, unsigned long pagetable)
 }
 #endif
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+
+void pmdp_splitting_flush(struct vm_area_struct *vma,
+                        unsigned long address,
+                        pmd_t *pmdp)
+{
+       if (!pmd_trans_splitting(*pmdp)) {
+               pmd_t pmd = pmd_mksplitting(*pmdp);
+               set_pmd_at(vma->vm_mm, address, pmdp, pmd);
+       }
+}
+
+#endif
+
+pmd_t mk_pmd(struct page *page, pgprot_t prot)
+{
+       pmd_t pmd;
+
+       pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot);
+
+       return pmd;
+}
+
+void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+               pmd_t *pmdp, pmd_t pmd)
+{
+       *pmdp = pmd;
+       flush_tlb_all();
+}
+
 void __init pagetable_init(void)
 {
        unsigned long vaddr;

diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index 936165d167e1a79655734ccac4ade9f5862e6082..94ad86d055c5e15df38d4748824870804a3aedfc 100644 (file)
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -377,6 +377,26 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
        EXIT_CRITICAL(flags);
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+
+int __init has_transparent_hugepage(void)
+{
+       unsigned int mask;
+       unsigned long flags;
+
+       ENTER_CRITICAL(flags);
+       write_c0_pagemask(PM_HUGE_MASK);
+       back_to_back_c0_hazard();
+       mask = read_c0_pagemask();
+       write_c0_pagemask(PM_DEFAULT_MASK);
+
+       EXIT_CRITICAL(flags);
+
+       return mask == PM_HUGE_MASK;
+}
+
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE  */
+
 static int __cpuinitdata ntlb;
 static int __init set_ntlb(char *str)
 {

diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 98b2b732005a131c8ed8ab57c5f3eb7e302d9d05..6af62a2fec21ada1d2d8456dbb46167102128440 100644 (file)
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -225,8 +225,9 @@ static void output_pgtable_bits_defines(void)
        pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
        pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT);
        pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT);
-#ifdef _PAGE_HUGE_SHIFT
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
        pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
+       pr_define("_PAGE_SPLITTING_SHIFT %d\n", _PAGE_SPLITTING_SHIFT);
 #endif
        if (cpu_has_rixi) {
 #ifdef _PAGE_NO_EXEC_SHIFT