arm64: mm: use appropriate ctors for page tables

For pte page, use pgtable_page_ctor(); for pmd page, use
pgtable_pmd_page_ctor(); and for the rest (pud, p4d and pgd),
don't use any.

For now, we don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK and
pgtable_pmd_page_ctor() is a nop. When we do in patch 3, we
make sure pmd is not folded so we won't mistakenly call
pgtable_pmd_page_ctor() on pud or p4d.

Acked-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Yu Zhao <yuzhao@google.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index e97f018ff740f8a1cea437aa88a652882412928b..66460c2074f692f23561dcd51bfc420a496e4261 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -97,7 +97,7 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 }
 EXPORT_SYMBOL(phys_mem_access_prot);
 
-static phys_addr_t __init early_pgtable_alloc(void)
+static phys_addr_t __init early_pgtable_alloc(int shift)
 {
        phys_addr_t phys;
        void *ptr;
@@ -174,7 +174,7 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
                                unsigned long end, phys_addr_t phys,
                                pgprot_t prot,
-                               phys_addr_t (*pgtable_alloc)(void),
+                               phys_addr_t (*pgtable_alloc)(int),
                                int flags)
 {
        unsigned long next;
@@ -184,7 +184,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
        if (pmd_none(pmd)) {
                phys_addr_t pte_phys;
                BUG_ON(!pgtable_alloc);
-               pte_phys = pgtable_alloc();
+               pte_phys = pgtable_alloc(PAGE_SHIFT);
                __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);
                pmd = READ_ONCE(*pmdp);
        }
@@ -208,7 +208,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
 
 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
                     phys_addr_t phys, pgprot_t prot,
-                    phys_addr_t (*pgtable_alloc)(void), int flags)
+                    phys_addr_t (*pgtable_alloc)(int), int flags)
 {
        unsigned long next;
        pmd_t *pmdp;
@@ -246,7 +246,7 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
                                unsigned long end, phys_addr_t phys,
                                pgprot_t prot,
-                               phys_addr_t (*pgtable_alloc)(void), int flags)
+                               phys_addr_t (*pgtable_alloc)(int), int flags)
 {
        unsigned long next;
        pud_t pud = READ_ONCE(*pudp);
@@ -258,7 +258,7 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
        if (pud_none(pud)) {
                phys_addr_t pmd_phys;
                BUG_ON(!pgtable_alloc);
-               pmd_phys = pgtable_alloc();
+               pmd_phys = pgtable_alloc(PMD_SHIFT);
                __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);
                pud = READ_ONCE(*pudp);
        }
@@ -294,7 +294,7 @@ static inline bool use_1G_block(unsigned long addr, unsigned long next,
 
 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
                           phys_addr_t phys, pgprot_t prot,
-                          phys_addr_t (*pgtable_alloc)(void),
+                          phys_addr_t (*pgtable_alloc)(int),
                           int flags)
 {
        unsigned long next;
@@ -304,7 +304,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
        if (pgd_none(pgd)) {
                phys_addr_t pud_phys;
                BUG_ON(!pgtable_alloc);
-               pud_phys = pgtable_alloc();
+               pud_phys = pgtable_alloc(PUD_SHIFT);
                __pgd_populate(pgdp, pud_phys, PUD_TYPE_TABLE);
                pgd = READ_ONCE(*pgdp);
        }
@@ -345,7 +345,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
 static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
                                 unsigned long virt, phys_addr_t size,
                                 pgprot_t prot,
-                                phys_addr_t (*pgtable_alloc)(void),
+                                phys_addr_t (*pgtable_alloc)(int),
                                 int flags)
 {
        unsigned long addr, length, end, next;
@@ -371,11 +371,23 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
        } while (pgdp++, addr = next, addr != end);
 }
 
-static phys_addr_t pgd_pgtable_alloc(void)
+static phys_addr_t pgd_pgtable_alloc(int shift)
 {
        void *ptr = (void *)__get_free_page(PGALLOC_GFP);
-       if (!ptr || !pgtable_page_ctor(virt_to_page(ptr)))
-               BUG();
+       BUG_ON(!ptr);
+
+       /*
+        * Call proper page table ctor in case later we need to
+        * call core mm functions like apply_to_page_range() on
+        * this pre-allocated page table.
+        *
+        * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is
+        * folded, and if so pgtable_pmd_page_ctor() becomes nop.
+        */
+       if (shift == PAGE_SHIFT)
+               BUG_ON(!pgtable_page_ctor(virt_to_page(ptr)));
+       else if (shift == PMD_SHIFT)
+               BUG_ON(!pgtable_pmd_page_ctor(virt_to_page(ptr)));
 
        /* Ensure the zeroed page is visible to the page table walker */
        dsb(ishst);