X-Git-Url: http://ftp.safe.ca/?a=blobdiff_plain;f=mm%2Fsparse-vmemmap.c;h=d9714bdcb4a387199a5875d9745999942a3aa4f8;hb=8051be5e614f3e3feccbe9e06b50e0b889740a93;hp=7bb7a4b96d74cb712a188c8d45d6c2cf447210f0;hpb=8f6aac419bd590f535fb110875a51f7db2b62b5b;p=safe%2Fjmp%2Flinux-2.6 diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 7bb7a4b..d9714bd 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -1,7 +1,7 @@ /* * Virtual Memory Map support * - * (C) 2007 sgi. Christoph Lameter . + * (C) 2007 sgi. Christoph Lameter. * * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn, * virt_to_page, page_address() to be implemented as a base offset @@ -14,21 +14,8 @@ * case the overhead consists of a few additional pages that are * allocated to create a view of memory for vmemmap. * - * Special Kconfig settings: - * - * CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP - * - * The architecture has its own functions to populate the memory - * map and provides a vmemmap_populate function. - * - * CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD - * - * The architecture provides functions to populate the pmd level - * of the vmemmap mappings. Allowing mappings using large pages - * where available. - * - * If neither are set then PAGE_SIZE mappings are generated which - * require one PTE/TLB per PAGE_SIZE chunk of the virtual memory map. + * The architecture is expected to provide a vmemmap_populate() function + * to instantiate the mapping. */ #include #include @@ -37,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -46,131 +34,127 @@ * or to back the page tables that are used to create the mapping. * Uses the main allocators if they are available, else bootmem. */ + +static void * __init_refok __earlyonly_bootmem_alloc(int node, + unsigned long size, + unsigned long align, + unsigned long goal) +{ + return __alloc_bootmem_node(NODE_DATA(node), size, align, goal); +} + + void * __meminit vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { - struct page *page = alloc_pages_node(node, + struct page *page; + + if (node_state(node, N_HIGH_MEMORY)) + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, get_order(size)); + else + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, + get_order(size)); if (page) return page_address(page); return NULL; } else - return __alloc_bootmem_node(NODE_DATA(node), size, size, + return __earlyonly_bootmem_alloc(node, size, size, __pa(MAX_DMA_ADDRESS)); } -#ifndef CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP void __meminit vmemmap_verify(pte_t *pte, int node, unsigned long start, unsigned long end) { unsigned long pfn = pte_pfn(*pte); int actual_node = early_pfn_to_nid(pfn); - if (actual_node != node) + if (node_distance(actual_node, node) > LOCAL_DISTANCE) printk(KERN_WARNING "[%lx-%lx] potential offnode " "page_structs\n", start, end - 1); } -#ifndef CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD -static int __meminit vmemmap_populate_pte(pmd_t *pmd, unsigned long addr, - unsigned long end, int node) +pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node) { - pte_t *pte; - - for (pte = pte_offset_kernel(pmd, addr); addr < end; - pte++, addr += PAGE_SIZE) - if (pte_none(*pte)) { - pte_t entry; - void *p = vmemmap_alloc_block(PAGE_SIZE, node); - if (!p) - return -ENOMEM; - - entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); - set_pte(pte, entry); - - } else - vmemmap_verify(pte, node, addr + PAGE_SIZE, end); + pte_t *pte = pte_offset_kernel(pmd, addr); + if (pte_none(*pte)) { + pte_t entry; + void *p = vmemmap_alloc_block(PAGE_SIZE, node); + if (!p) + return NULL; + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); + set_pte_at(&init_mm, addr, pte, entry); + } + return pte; +} - return 0; +pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) +{ + pmd_t *pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) { + void *p = vmemmap_alloc_block(PAGE_SIZE, node); + if (!p) + return NULL; + pmd_populate_kernel(&init_mm, pmd, p); + } + return pmd; } -int __meminit vmemmap_populate_pmd(pud_t *pud, unsigned long addr, - unsigned long end, int node) +pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node) { - pmd_t *pmd; - int error = 0; - unsigned long next; - - for (pmd = pmd_offset(pud, addr); addr < end && !error; - pmd++, addr = next) { - if (pmd_none(*pmd)) { - void *p = vmemmap_alloc_block(PAGE_SIZE, node); - if (!p) - return -ENOMEM; - - pmd_populate_kernel(&init_mm, pmd, p); - } else - vmemmap_verify((pte_t *)pmd, node, - pmd_addr_end(addr, end), end); - next = pmd_addr_end(addr, end); - error = vmemmap_populate_pte(pmd, addr, next, node); + pud_t *pud = pud_offset(pgd, addr); + if (pud_none(*pud)) { + void *p = vmemmap_alloc_block(PAGE_SIZE, node); + if (!p) + return NULL; + pud_populate(&init_mm, pud, p); } - return error; + return pud; } -#endif /* CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD */ -static int __meminit vmemmap_populate_pud(pgd_t *pgd, unsigned long addr, - unsigned long end, int node) +pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) { - pud_t *pud; - int error = 0; - unsigned long next; - - for (pud = pud_offset(pgd, addr); addr < end && !error; - pud++, addr = next) { - if (pud_none(*pud)) { - void *p = vmemmap_alloc_block(PAGE_SIZE, node); - if (!p) - return -ENOMEM; - - pud_populate(&init_mm, pud, p); - } - next = pud_addr_end(addr, end); - error = vmemmap_populate_pmd(pud, addr, next, node); + pgd_t *pgd = pgd_offset_k(addr); + if (pgd_none(*pgd)) { + void *p = vmemmap_alloc_block(PAGE_SIZE, node); + if (!p) + return NULL; + pgd_populate(&init_mm, pgd, p); } - return error; + return pgd; } -int __meminit vmemmap_populate(struct page *start_page, - unsigned long nr, int node) +int __meminit vmemmap_populate_basepages(struct page *start_page, + unsigned long size, int node) { - pgd_t *pgd; unsigned long addr = (unsigned long)start_page; - unsigned long end = (unsigned long)(start_page + nr); - unsigned long next; - int error = 0; - - printk(KERN_DEBUG "[%lx-%lx] Virtual memory section" - " (%ld pages) node %d\n", addr, end - 1, nr, node); - - for (pgd = pgd_offset_k(addr); addr < end && !error; - pgd++, addr = next) { - if (pgd_none(*pgd)) { - void *p = vmemmap_alloc_block(PAGE_SIZE, node); - if (!p) - return -ENOMEM; - - pgd_populate(&init_mm, pgd, p); - } - next = pgd_addr_end(addr,end); - error = vmemmap_populate_pud(pgd, addr, next, node); + unsigned long end = (unsigned long)(start_page + size); + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + for (; addr < end; addr += PAGE_SIZE) { + pgd = vmemmap_pgd_populate(addr, node); + if (!pgd) + return -ENOMEM; + pud = vmemmap_pud_populate(pgd, addr, node); + if (!pud) + return -ENOMEM; + pmd = vmemmap_pmd_populate(pud, addr, node); + if (!pmd) + return -ENOMEM; + pte = vmemmap_pte_populate(pmd, addr, node); + if (!pte) + return -ENOMEM; + vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); } - return error; + + return 0; } -#endif /* !CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP */ -struct page __init *sparse_early_mem_map_populate(unsigned long pnum, int nid) +struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) { struct page *map = pfn_to_page(pnum * PAGES_PER_SECTION); int error = vmemmap_populate(map, PAGES_PER_SECTION, nid);