x86: support 1GB hugepages with get_user_pages_lockless()

Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 6f733121f32e6dcf18106e82eebd0b46a6e3fc26..3085f25b4355de827fee53d8851180a844777517 100644 (file)
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -124,7 +124,7 @@ static noinline int gup_huge_pmd(pmd_t pmd, unsigned long addr,
 
        refs = 0;
        head = pte_page(pte);
-       page = head + ((addr & ~HPAGE_MASK) >> PAGE_SHIFT);
+       page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
        do {
                VM_BUG_ON(compound_head(page) != head);
                pages[*nr] = page;
@@ -162,6 +162,38 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
        return 1;
 }
 
+static noinline int gup_huge_pud(pud_t pud, unsigned long addr,
+               unsigned long end, int write, struct page **pages, int *nr)
+{
+       unsigned long mask;
+       pte_t pte = *(pte_t *)&pud;
+       struct page *head, *page;
+       int refs;
+
+       mask = _PAGE_PRESENT|_PAGE_USER;
+       if (write)
+               mask |= _PAGE_RW;
+       if ((pte_val(pte) & mask) != mask)
+               return 0;
+       /* hugepages are never "special" */
+       VM_BUG_ON(pte_val(pte) & _PAGE_SPECIAL);
+       VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
+
+       refs = 0;
+       head = pte_page(pte);
+       page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
+       do {
+               VM_BUG_ON(compound_head(page) != head);
+               pages[*nr] = page;
+               (*nr)++;
+               page++;
+               refs++;
+       } while (addr += PAGE_SIZE, addr != end);
+       get_head_page_multiple(head, refs);
+
+       return 1;
+}
+
 static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
                        int write, struct page **pages, int *nr)
 {
@@ -175,8 +207,13 @@ static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
                next = pud_addr_end(addr, end);
                if (pud_none(pud))
                        return 0;
-               if (!gup_pmd_range(pud, addr, next, write, pages, nr))
-                       return 0;
+               if (unlikely(pud_large(pud))) {
+                       if (!gup_huge_pud(pud, addr, next, write, pages, nr))
+                               return 0;
+               } else {
+                       if (!gup_pmd_range(pud, addr, next, write, pages, nr))
+                               return 0;
+               }
        } while (pudp++, addr = next, addr != end);
 
        return 1;