mm: pass the vmem_altmap to vmemmap_free

We can just pass this on instead of having to do a radix tree lookup
without proper locking a few levels into the callchain.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index ec8952ff13bebe617ee08b4ad583869ec012365f..0b1f13e0b4b36d98d0b21e4a5eede0754f2c5d21 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -696,7 +696,8 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
        return 0;
 }
 #endif /* CONFIG_ARM64_64K_PAGES */
-void vmemmap_free(unsigned long start, unsigned long end)
+void vmemmap_free(unsigned long start, unsigned long end,
+               struct vmem_altmap *altmap)
 {
 }
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */

diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c
index 1555aecaaf85b887691466896bb43a48ef0cfd2d..5ea0d8d0968bc58d89a08d9e654a1d03ed3c7f14 100644 (file)
--- a/arch/ia64/mm/discontig.c
+++ b/arch/ia64/mm/discontig.c
@@ -760,7 +760,8 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
        return vmemmap_populate_basepages(start, end, node);
 }
 
-void vmemmap_free(unsigned long start, unsigned long end)
+void vmemmap_free(unsigned long start, unsigned long end,
+               struct vmem_altmap *altmap)
 {
 }
 #endif

diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index 779b74a96b8fd828c82e03df0e066006234100cb..db7d4e09215767c8f0389a0519231f1f668d7557 100644 (file)
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -254,7 +254,8 @@ static unsigned long vmemmap_list_free(unsigned long start)
        return vmem_back->phys;
 }
 
-void __ref vmemmap_free(unsigned long start, unsigned long end)
+void __ref vmemmap_free(unsigned long start, unsigned long end,
+               struct vmem_altmap *altmap)
 {
        unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
        unsigned long page_order = get_order(page_size);
@@ -265,7 +266,6 @@ void __ref vmemmap_free(unsigned long start, unsigned long end)
 
        for (; start < end; start += page_size) {
                unsigned long nr_pages, addr;
-               struct vmem_altmap *altmap;
                struct page *section_base;
                struct page *page;
 
@@ -285,7 +285,6 @@ void __ref vmemmap_free(unsigned long start, unsigned long end)
                section_base = pfn_to_page(vmemmap_section_start(start));
                nr_pages = 1 << page_order;
 
-               altmap = to_vmem_altmap((unsigned long) section_base);
                if (altmap) {
                        vmem_altmap_free(altmap, nr_pages);
                } else if (PageReserved(page)) {

diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index c44ef0e7c466c395cabc05b0a77fc408928e4884..db55561c598130475fc3828ce54c9ad394788dd0 100644 (file)
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -297,7 +297,8 @@ out:
        return ret;
 }
 
-void vmemmap_free(unsigned long start, unsigned long end)
+void vmemmap_free(unsigned long start, unsigned long end,
+               struct vmem_altmap *altmap)
 {
 }
 

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 42d27a1a042ad7afcdda3fb524fc6779e8b8ca31..995f9490334dd0001ceb0788d15b93e9f80821a4 100644 (file)
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2671,7 +2671,8 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
        return 0;
 }
 
-void vmemmap_free(unsigned long start, unsigned long end)
+void vmemmap_free(unsigned long start, unsigned long end,
+               struct vmem_altmap *altmap)
 {
 }
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 3c046618cc7e01e35c63a6353ef30642032ec9ec..0cab4b5b59ba8407e312003d4cdee9a58a6f6dbc 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -800,11 +800,11 @@ int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
 
 #define PAGE_INUSE 0xFD
 
-static void __meminit free_pagetable(struct page *page, int order)
+static void __meminit free_pagetable(struct page *page, int order,
+               struct vmem_altmap *altmap)
 {
        unsigned long magic;
        unsigned int nr_pages = 1 << order;
-       struct vmem_altmap *altmap = to_vmem_altmap((unsigned long) page);
 
        if (altmap) {
                vmem_altmap_free(altmap, nr_pages);
@@ -826,7 +826,8 @@ static void __meminit free_pagetable(struct page *page, int order)
                free_pages((unsigned long)page_address(page), order);
 }
 
-static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
+static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd,
+               struct vmem_altmap *altmap)
 {
        pte_t *pte;
        int i;
@@ -838,13 +839,14 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
        }
 
        /* free a pte talbe */
-       free_pagetable(pmd_page(*pmd), 0);
+       free_pagetable(pmd_page(*pmd), 0, altmap);
        spin_lock(&init_mm.page_table_lock);
        pmd_clear(pmd);
        spin_unlock(&init_mm.page_table_lock);
 }
 
-static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)
+static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud,
+               struct vmem_altmap *altmap)
 {
        pmd_t *pmd;
        int i;
@@ -856,13 +858,14 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)
        }
 
        /* free a pmd talbe */
-       free_pagetable(pud_page(*pud), 0);
+       free_pagetable(pud_page(*pud), 0, altmap);
        spin_lock(&init_mm.page_table_lock);
        pud_clear(pud);
        spin_unlock(&init_mm.page_table_lock);
 }
 
-static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d)
+static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d,
+               struct vmem_altmap *altmap)
 {
        pud_t *pud;
        int i;
@@ -874,7 +877,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d)
        }
 
        /* free a pud talbe */
-       free_pagetable(p4d_page(*p4d), 0);
+       free_pagetable(p4d_page(*p4d), 0, altmap);
        spin_lock(&init_mm.page_table_lock);
        p4d_clear(p4d);
        spin_unlock(&init_mm.page_table_lock);
@@ -882,7 +885,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d)
 
 static void __meminit
 remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
-                bool direct)
+                struct vmem_altmap *altmap, bool direct)
 {
        unsigned long next, pages = 0;
        pte_t *pte;
@@ -913,7 +916,7 @@ remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
                         * freed when offlining, or simplely not in use.
                         */
                        if (!direct)
-                               free_pagetable(pte_page(*pte), 0);
+                               free_pagetable(pte_page(*pte), 0, altmap);
 
                        spin_lock(&init_mm.page_table_lock);
                        pte_clear(&init_mm, addr, pte);
@@ -936,7 +939,7 @@ remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
 
                        page_addr = page_address(pte_page(*pte));
                        if (!memchr_inv(page_addr, PAGE_INUSE, PAGE_SIZE)) {
-                               free_pagetable(pte_page(*pte), 0);
+                               free_pagetable(pte_page(*pte), 0, altmap);
 
                                spin_lock(&init_mm.page_table_lock);
                                pte_clear(&init_mm, addr, pte);
@@ -953,7 +956,7 @@ remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
 
 static void __meminit
 remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
-                bool direct)
+                bool direct, struct vmem_altmap *altmap)
 {
        unsigned long next, pages = 0;
        pte_t *pte_base;
@@ -972,7 +975,8 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
                            IS_ALIGNED(next, PMD_SIZE)) {
                                if (!direct)
                                        free_pagetable(pmd_page(*pmd),
-                                                      get_order(PMD_SIZE));
+                                                      get_order(PMD_SIZE),
+                                                      altmap);
 
                                spin_lock(&init_mm.page_table_lock);
                                pmd_clear(pmd);
@@ -986,7 +990,8 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
                                if (!memchr_inv(page_addr, PAGE_INUSE,
                                                PMD_SIZE)) {
                                        free_pagetable(pmd_page(*pmd),
-                                                      get_order(PMD_SIZE));
+                                                      get_order(PMD_SIZE),
+                                                      altmap);
 
                                        spin_lock(&init_mm.page_table_lock);
                                        pmd_clear(pmd);
@@ -998,8 +1003,8 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
                }
 
                pte_base = (pte_t *)pmd_page_vaddr(*pmd);
-               remove_pte_table(pte_base, addr, next, direct);
-               free_pte_table(pte_base, pmd);
+               remove_pte_table(pte_base, addr, next, altmap, direct);
+               free_pte_table(pte_base, pmd, altmap);
        }
 
        /* Call free_pmd_table() in remove_pud_table(). */
@@ -1009,7 +1014,7 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
 
 static void __meminit
 remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
-                bool direct)
+                struct vmem_altmap *altmap, bool direct)
 {
        unsigned long next, pages = 0;
        pmd_t *pmd_base;
@@ -1028,7 +1033,8 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
                            IS_ALIGNED(next, PUD_SIZE)) {
                                if (!direct)
                                        free_pagetable(pud_page(*pud),
-                                                      get_order(PUD_SIZE));
+                                                      get_order(PUD_SIZE),
+                                                      altmap);
 
                                spin_lock(&init_mm.page_table_lock);
                                pud_clear(pud);
@@ -1042,7 +1048,8 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
                                if (!memchr_inv(page_addr, PAGE_INUSE,
                                                PUD_SIZE)) {
                                        free_pagetable(pud_page(*pud),
-                                                      get_order(PUD_SIZE));
+                                                      get_order(PUD_SIZE),
+                                                      altmap);
 
                                        spin_lock(&init_mm.page_table_lock);
                                        pud_clear(pud);
@@ -1054,8 +1061,8 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
                }
 
                pmd_base = pmd_offset(pud, 0);
-               remove_pmd_table(pmd_base, addr, next, direct);
-               free_pmd_table(pmd_base, pud);
+               remove_pmd_table(pmd_base, addr, next, direct, altmap);
+               free_pmd_table(pmd_base, pud, altmap);
        }
 
        if (direct)
@@ -1064,7 +1071,7 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
 
 static void __meminit
 remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
-                bool direct)
+                struct vmem_altmap *altmap, bool direct)
 {
        unsigned long next, pages = 0;
        pud_t *pud_base;
@@ -1080,14 +1087,14 @@ remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
                BUILD_BUG_ON(p4d_large(*p4d));
 
                pud_base = pud_offset(p4d, 0);
-               remove_pud_table(pud_base, addr, next, direct);
+               remove_pud_table(pud_base, addr, next, altmap, direct);
                /*
                 * For 4-level page tables we do not want to free PUDs, but in the
                 * 5-level case we should free them. This code will have to change
                 * to adapt for boot-time switching between 4 and 5 level page tables.
                 */
                if (CONFIG_PGTABLE_LEVELS == 5)
-                       free_pud_table(pud_base, p4d);
+                       free_pud_table(pud_base, p4d, altmap);
        }
 
        if (direct)
@@ -1096,7 +1103,8 @@ remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
 
 /* start and end are both virtual address. */
 static void __meminit
-remove_pagetable(unsigned long start, unsigned long end, bool direct)
+remove_pagetable(unsigned long start, unsigned long end, bool direct,
+               struct vmem_altmap *altmap)
 {
        unsigned long next;
        unsigned long addr;
@@ -1111,15 +1119,16 @@ remove_pagetable(unsigned long start, unsigned long end, bool direct)
                        continue;
 
                p4d = p4d_offset(pgd, 0);
-               remove_p4d_table(p4d, addr, next, direct);
+               remove_p4d_table(p4d, addr, next, altmap, direct);
        }
 
        flush_tlb_all();
 }
 
-void __ref vmemmap_free(unsigned long start, unsigned long end)
+void __ref vmemmap_free(unsigned long start, unsigned long end,
+               struct vmem_altmap *altmap)
 {
-       remove_pagetable(start, end, false);
+       remove_pagetable(start, end, false, altmap);
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
@@ -1129,7 +1138,7 @@ kernel_physical_mapping_remove(unsigned long start, unsigned long end)
        start = (unsigned long)__va(start);
        end = (unsigned long)__va(end);
 
-       remove_pagetable(start, end, true);
+       remove_pagetable(start, end, true, NULL);
 }
 
 int __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)

diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index e71927d0d46bd98db16e2ca2c07389443a2def47..20dd98ad44a028c71904baa7b672cc10553dcb3c 100644 (file)
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -331,7 +331,7 @@ extern void remove_memory(int nid, u64 start, u64 size);
 extern int sparse_add_one_section(struct pglist_data *pgdat,
                unsigned long start_pfn, struct vmem_altmap *altmap);
 extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
-               unsigned long map_offset);
+               unsigned long map_offset, struct vmem_altmap *altmap);
 extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map,
                                          unsigned long pnum);
 extern bool allow_online_pfn_range(int nid, unsigned long pfn, unsigned long nr_pages,

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 2f3a7ebecbe2f7b4757e8726071fdabacc370460..9d4cd4c1dc6df158c1675ae4df5e4b0a797abe5c 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2561,7 +2561,8 @@ int vmemmap_populate(unsigned long start, unsigned long end, int node,
                struct vmem_altmap *altmap);
 void vmemmap_populate_print_last(void);
 #ifdef CONFIG_MEMORY_HOTPLUG
-void vmemmap_free(unsigned long start, unsigned long end);
+void vmemmap_free(unsigned long start, unsigned long end,
+               struct vmem_altmap *altmap);
 #endif
 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
                                  unsigned long nr_pages);

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index eae6bf47caf7a5cf2aa1db464bd206fbf1d86901..a8dde9734120ce5d140da841f41f4962b9eee942 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -536,7 +536,7 @@ static void __remove_zone(struct zone *zone, unsigned long start_pfn)
 }
 
 static int __remove_section(struct zone *zone, struct mem_section *ms,
-               unsigned long map_offset)
+               unsigned long map_offset, struct vmem_altmap *altmap)
 {
        unsigned long start_pfn;
        int scn_nr;
@@ -553,7 +553,7 @@ static int __remove_section(struct zone *zone, struct mem_section *ms,
        start_pfn = section_nr_to_pfn((unsigned long)scn_nr);
        __remove_zone(zone, start_pfn);
 
-       sparse_remove_one_section(zone, ms, map_offset);
+       sparse_remove_one_section(zone, ms, map_offset, altmap);
        return 0;
 }
 
@@ -607,7 +607,8 @@ int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
        for (i = 0; i < sections_to_remove; i++) {
                unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
 
-               ret = __remove_section(zone, __pfn_to_section(pfn), map_offset);
+               ret = __remove_section(zone, __pfn_to_section(pfn), map_offset,
+                               altmap);
                map_offset = 0;
                if (ret)
                        break;

diff --git a/mm/sparse.c b/mm/sparse.c
index 5f4a0dac7836e2e87aef47d6bc23026ee93338e6..06130c13dc99221ba421ff3483ba24346cc712b4 100644 (file)
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -685,12 +685,13 @@ static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
        /* This will make the necessary allocations eventually. */
        return sparse_mem_map_populate(pnum, nid, altmap);
 }
-static void __kfree_section_memmap(struct page *memmap)
+static void __kfree_section_memmap(struct page *memmap,
+               struct vmem_altmap *altmap)
 {
        unsigned long start = (unsigned long)memmap;
        unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
 
-       vmemmap_free(start, end);
+       vmemmap_free(start, end, altmap);
 }
 #ifdef CONFIG_MEMORY_HOTREMOVE
 static void free_map_bootmem(struct page *memmap)
@@ -698,7 +699,7 @@ static void free_map_bootmem(struct page *memmap)
        unsigned long start = (unsigned long)memmap;
        unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
 
-       vmemmap_free(start, end);
+       vmemmap_free(start, end, NULL);
 }
 #endif /* CONFIG_MEMORY_HOTREMOVE */
 #else
@@ -729,7 +730,8 @@ static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
        return __kmalloc_section_memmap();
 }
 
-static void __kfree_section_memmap(struct page *memmap)
+static void __kfree_section_memmap(struct page *memmap,
+               struct vmem_altmap *altmap)
 {
        if (is_vmalloc_addr(memmap))
                vfree(memmap);
@@ -798,7 +800,7 @@ int __meminit sparse_add_one_section(struct pglist_data *pgdat,
                return -ENOMEM;
        usemap = __kmalloc_section_usemap();
        if (!usemap) {
-               __kfree_section_memmap(memmap);
+               __kfree_section_memmap(memmap, altmap);
                return -ENOMEM;
        }
 
@@ -820,7 +822,7 @@ out:
        pgdat_resize_unlock(pgdat, &flags);
        if (ret <= 0) {
                kfree(usemap);
-               __kfree_section_memmap(memmap);
+               __kfree_section_memmap(memmap, altmap);
        }
        return ret;
 }
@@ -847,7 +849,8 @@ static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
 }
 #endif
 
-static void free_section_usemap(struct page *memmap, unsigned long *usemap)
+static void free_section_usemap(struct page *memmap, unsigned long *usemap,
+               struct vmem_altmap *altmap)
 {
        struct page *usemap_page;
 
@@ -861,7 +864,7 @@ static void free_section_usemap(struct page *memmap, unsigned long *usemap)
        if (PageSlab(usemap_page) || PageCompound(usemap_page)) {
                kfree(usemap);
                if (memmap)
-                       __kfree_section_memmap(memmap);
+                       __kfree_section_memmap(memmap, altmap);
                return;
        }
 
@@ -875,7 +878,7 @@ static void free_section_usemap(struct page *memmap, unsigned long *usemap)
 }
 
 void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
-               unsigned long map_offset)
+               unsigned long map_offset, struct vmem_altmap *altmap)
 {
        struct page *memmap = NULL;
        unsigned long *usemap = NULL, flags;
@@ -893,7 +896,7 @@ void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
 
        clear_hwpoisoned_pages(memmap + map_offset,
                        PAGES_PER_SECTION - map_offset);
-       free_section_usemap(memmap, usemap);
+       free_section_usemap(memmap, usemap, altmap);
 }
 #endif /* CONFIG_MEMORY_HOTREMOVE */
 #endif /* CONFIG_MEMORY_HOTPLUG */