X-Git-Url: http://ftp.safe.ca/?a=blobdiff_plain;f=mm%2Fvmalloc.c;h=2e01af365848eb3d8485369d2fc4d9c4dad2a05c;hb=74beb9db77930be476b267ec8518a642f39a04bf;hp=35f8553f893a7ffe0416732827c71385646c4ea3;hpb=833423143c3a7c6545e409d65febd0d92deb351b;p=safe%2Fjmp%2Flinux-2.6 diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 35f8553..2e01af3 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -24,6 +24,9 @@ DEFINE_RWLOCK(vmlist_lock); struct vm_struct *vmlist; +static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, + int node); + static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end) { pte_t *pte; @@ -65,12 +68,12 @@ static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr, } while (pud++, addr = next, addr != end); } -void unmap_vm_area(struct vm_struct *area) +void unmap_kernel_range(unsigned long addr, unsigned long size) { pgd_t *pgd; unsigned long next; - unsigned long addr = (unsigned long) area->addr; - unsigned long end = addr + area->size; + unsigned long start = addr; + unsigned long end = addr + size; BUG_ON(addr >= end); pgd = pgd_offset_k(addr); @@ -81,7 +84,12 @@ void unmap_vm_area(struct vm_struct *area) continue; vunmap_pud_range(pgd, addr, next); } while (pgd++, addr = next, addr != end); - flush_tlb_kernel_range((unsigned long) area->addr, end); + flush_tlb_kernel_range(start, end); +} + +static void unmap_vm_area(struct vm_struct *area) +{ + unmap_kernel_range((unsigned long)area->addr, area->size); } static int vmap_pte_range(pmd_t *pmd, unsigned long addr, @@ -156,14 +164,17 @@ int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages) flush_cache_vmap((unsigned long) area->addr, end); return err; } +EXPORT_SYMBOL_GPL(map_vm_area); -struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags, - unsigned long start, unsigned long end, int node) +static struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags, + unsigned long start, unsigned long end, + int node, gfp_t gfp_mask) { struct vm_struct **p, *tmp, *area; unsigned long align = 1; unsigned long addr; + BUG_ON(in_interrupt()); if (flags & VM_IOREMAP) { int bit = fls(size); @@ -176,15 +187,13 @@ struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags, } addr = ALIGN(start, align); size = PAGE_ALIGN(size); - - area = kmalloc_node(sizeof(*area), GFP_KERNEL, node); - if (unlikely(!area)) + if (unlikely(!size)) return NULL; - if (unlikely(!size)) { - kfree (area); + area = kmalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); + + if (unlikely(!area)) return NULL; - } /* * We always allocate a guard page. @@ -233,12 +242,12 @@ out: struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, unsigned long start, unsigned long end) { - return __get_vm_area_node(size, flags, start, end, -1); + return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL); } +EXPORT_SYMBOL_GPL(__get_vm_area); /** * get_vm_area - reserve a contingous kernel virtual area - * * @size: size of the area * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC * @@ -251,9 +260,11 @@ struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END); } -struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, int node) +struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, + int node, gfp_t gfp_mask) { - return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node); + return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node, + gfp_mask); } /* Caller must hold vmlist_lock */ @@ -270,7 +281,7 @@ static struct vm_struct *__find_vm_area(void *addr) } /* Caller must hold vmlist_lock */ -struct vm_struct *__remove_vm_area(void *addr) +static struct vm_struct *__remove_vm_area(void *addr) { struct vm_struct **p, *tmp; @@ -293,7 +304,6 @@ found: /** * remove_vm_area - find and remove a contingous kernel virtual area - * * @addr: base address * * Search for the kernel VM area starting at @addr, and remove it. @@ -309,7 +319,7 @@ struct vm_struct *remove_vm_area(void *addr) return v; } -void __vunmap(void *addr, int deallocate_pages) +static void __vunmap(void *addr, int deallocate_pages) { struct vm_struct *area; @@ -330,6 +340,8 @@ void __vunmap(void *addr, int deallocate_pages) return; } + debug_check_no_locks_freed(addr, area->size); + if (deallocate_pages) { int i; @@ -338,7 +350,7 @@ void __vunmap(void *addr, int deallocate_pages) __free_page(area->pages[i]); } - if (area->nr_pages > PAGE_SIZE/sizeof(struct page *)) + if (area->flags & VM_VPAGES) vfree(area->pages); else kfree(area->pages); @@ -350,7 +362,6 @@ void __vunmap(void *addr, int deallocate_pages) /** * vfree - release memory allocated by vmalloc() - * * @addr: memory base address * * Free the virtually contiguous memory area starting at @addr, as @@ -368,7 +379,6 @@ EXPORT_SYMBOL(vfree); /** * vunmap - release virtual mapping obtained by vmap() - * * @addr: memory base address * * Free the virtually contiguous memory area starting at @addr, @@ -385,7 +395,6 @@ EXPORT_SYMBOL(vunmap); /** * vmap - map an array of pages into virtually contiguous space - * * @pages: array of page pointers * @count: number of pages to map * @flags: vm_area->flags @@ -425,17 +434,21 @@ void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, area->nr_pages = nr_pages; /* Please note that the recursion is strictly bounded. */ - if (array_size > PAGE_SIZE) - pages = __vmalloc_node(array_size, gfp_mask, PAGE_KERNEL, node); - else - pages = kmalloc_node(array_size, (gfp_mask & ~__GFP_HIGHMEM), node); + if (array_size > PAGE_SIZE) { + pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO, + PAGE_KERNEL, node); + area->flags |= VM_VPAGES; + } else { + pages = kmalloc_node(array_size, + (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO, + node); + } area->pages = pages; if (!area->pages) { remove_vm_area(area->addr); kfree(area); return NULL; } - memset(area->pages, 0, array_size); for (i = 0; i < area->nr_pages; i++) { if (node < 0) @@ -465,7 +478,6 @@ void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot) /** * __vmalloc_node - allocate virtually contiguous memory - * * @size: allocation size * @gfp_mask: flags for the page level allocator * @prot: protection mask for the allocated pages @@ -475,8 +487,8 @@ void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot) * allocator with @gfp_mask flags. Map them into contiguous * kernel virtual space, using a pagetable protection of @prot. */ -void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, - int node) +static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, + int node) { struct vm_struct *area; @@ -484,13 +496,12 @@ void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, if (!size || (size >> PAGE_SHIFT) > num_physpages) return NULL; - area = get_vm_area_node(size, VM_ALLOC, node); + area = get_vm_area_node(size, VM_ALLOC, node, gfp_mask); if (!area) return NULL; return __vmalloc_area_node(area, gfp_mask, prot, node); } -EXPORT_SYMBOL(__vmalloc_node); void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) { @@ -500,13 +511,11 @@ EXPORT_SYMBOL(__vmalloc); /** * vmalloc - allocate virtually contiguous memory - * * @size: allocation size - * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * - * For tight cotrol over page level allocator and protection flags + * For tight control over page level allocator and protection flags * use __vmalloc() instead. */ void *vmalloc(unsigned long size) @@ -516,11 +525,11 @@ void *vmalloc(unsigned long size) EXPORT_SYMBOL(vmalloc); /** - * vmalloc_user - allocate virtually contiguous memory which has - * been zeroed so it can be mapped to userspace without - * leaking data. + * vmalloc_user - allocate zeroed virtually contiguous memory for userspace + * @size: allocation size * - * @size: allocation size + * The resulting memory area is zeroed so it can be mapped to userspace + * without leaking data. */ void *vmalloc_user(unsigned long size) { @@ -528,25 +537,25 @@ void *vmalloc_user(unsigned long size) void *ret; ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL); - write_lock(&vmlist_lock); - area = __find_vm_area(ret); - area->flags |= VM_USERMAP; - write_unlock(&vmlist_lock); - + if (ret) { + write_lock(&vmlist_lock); + area = __find_vm_area(ret); + area->flags |= VM_USERMAP; + write_unlock(&vmlist_lock); + } return ret; } EXPORT_SYMBOL(vmalloc_user); /** * vmalloc_node - allocate memory on a specific node - * * @size: allocation size * @node: numa node * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * - * For tight cotrol over page level allocator and protection flags + * For tight control over page level allocator and protection flags * use __vmalloc() instead. */ void *vmalloc_node(unsigned long size, int node) @@ -561,14 +570,13 @@ EXPORT_SYMBOL(vmalloc_node); /** * vmalloc_exec - allocate virtually contiguous, executable memory - * * @size: allocation size * * Kernel-internal function to allocate enough pages to cover @size * the page level allocator and map them into contiguous and * executable kernel virtual space. * - * For tight cotrol over page level allocator and protection flags + * For tight control over page level allocator and protection flags * use __vmalloc() instead. */ @@ -577,9 +585,16 @@ void *vmalloc_exec(unsigned long size) return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC); } +#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) +#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL +#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) +#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL +#else +#define GFP_VMALLOC32 GFP_KERNEL +#endif + /** * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) - * * @size: allocation size * * Allocate enough 32bit PA addressable pages to cover @size from the @@ -587,28 +602,29 @@ void *vmalloc_exec(unsigned long size) */ void *vmalloc_32(unsigned long size) { - return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL); + return __vmalloc(size, GFP_VMALLOC32, PAGE_KERNEL); } EXPORT_SYMBOL(vmalloc_32); /** - * vmalloc_32_user - allocate virtually contiguous memory (32bit - * addressable) which is zeroed so it can be - * mapped to userspace without leaking data. - * + * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory * @size: allocation size + * + * The resulting memory area is 32bit addressable and zeroed so it can be + * mapped to userspace without leaking data. */ void *vmalloc_32_user(unsigned long size) { struct vm_struct *area; void *ret; - ret = __vmalloc(size, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL); - write_lock(&vmlist_lock); - area = __find_vm_area(ret); - area->flags |= VM_USERMAP; - write_unlock(&vmlist_lock); - + ret = __vmalloc(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL); + if (ret) { + write_lock(&vmlist_lock); + area = __find_vm_area(ret); + area->flags |= VM_USERMAP; + write_unlock(&vmlist_lock); + } return ret; } EXPORT_SYMBOL(vmalloc_32_user); @@ -690,7 +706,6 @@ finished: /** * remap_vmalloc_range - map vmalloc pages to userspace - * * @vma: vma to cover (map full range of vma) * @addr: vmalloc memory * @pgoff: number of pages into addr before first page to map @@ -700,7 +715,7 @@ finished: * that it is big enough to cover the vma. Will return failure if * that criteria isn't met. * - * Similar to remap_pfn_range (see mm/memory.c) + * Similar to remap_pfn_range() (see mm/memory.c) */ int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, unsigned long pgoff) @@ -748,3 +763,63 @@ out_einval_locked: } EXPORT_SYMBOL(remap_vmalloc_range); +/* + * Implement a stub for vmalloc_sync_all() if the architecture chose not to + * have one. + */ +void __attribute__((weak)) vmalloc_sync_all(void) +{ +} + + +static int f(pte_t *pte, struct page *pmd_page, unsigned long addr, void *data) +{ + /* apply_to_page_range() does all the hard work. */ + return 0; +} + +/** + * alloc_vm_area - allocate a range of kernel address space + * @size: size of the area + * @returns: NULL on failure, vm_struct on success + * + * This function reserves a range of kernel address space, and + * allocates pagetables to map that range. No actual mappings + * are created. If the kernel address space is not shared + * between processes, it syncs the pagetable across all + * processes. + */ +struct vm_struct *alloc_vm_area(size_t size) +{ + struct vm_struct *area; + + area = get_vm_area(size, VM_IOREMAP); + if (area == NULL) + return NULL; + + /* + * This ensures that page tables are constructed for this region + * of kernel virtual address space and mapped into init_mm. + */ + if (apply_to_page_range(&init_mm, (unsigned long)area->addr, + area->size, f, NULL)) { + free_vm_area(area); + return NULL; + } + + /* Make sure the pagetables are constructed in process kernel + mappings */ + vmalloc_sync_all(); + + return area; +} +EXPORT_SYMBOL_GPL(alloc_vm_area); + +void free_vm_area(struct vm_struct *area) +{ + struct vm_struct *ret; + ret = remove_vm_area(area->addr); + BUG_ON(ret != area); + kfree(area); +} +EXPORT_SYMBOL_GPL(free_vm_area);