Merge branch 'for-linus' of git://github.com/at91linux/linux-2.6-at91

[safe/jmp/linux-2.6] / mm / bootmem.c

diff --git a/mm/bootmem.c b/mm/bootmem.c
index d7140c0..555d5d2 100644 (file)
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -12,6 +12,7 @@
 #include <linux/pfn.h>
 #include <linux/bootmem.h>
 #include <linux/module.h>
+#include <linux/kmemleak.h>
 
 #include <asm/bug.h>
 #include <asm/io.h>
@@ -37,16 +38,6 @@ static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
 
 static int bootmem_debug;
 
-/*
- * If an arch needs to apply workarounds to bootmem allocation, it can
- * set CONFIG_HAVE_ARCH_BOOTMEM and define a wrapper around
- * __alloc_bootmem_core().
- */
-#ifndef CONFIG_HAVE_ARCH_BOOTMEM
-#define alloc_bootmem_core(bdata, size, align, goal, limit)            \
-       __alloc_bootmem_core((bdata), (size), (align), (goal), (limit))
-#endif
-
 static int __init bootmem_debug_setup(char *buf)
 {
        bootmem_debug = 1;
@@ -345,6 +336,8 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 {
        unsigned long start, end;
 
+       kmemleak_free_part(__va(physaddr), size);
+
        start = PFN_UP(physaddr);
        end = PFN_DOWN(physaddr + size);
 
@@ -364,6 +357,8 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
 {
        unsigned long start, end;
 
+       kmemleak_free_part(__va(addr), size);
+
        start = PFN_UP(addr);
        end = PFN_DOWN(addr + size);
 
@@ -436,9 +431,9 @@ static unsigned long align_off(struct bootmem_data *bdata, unsigned long off,
        return ALIGN(base + off, align) - base;
 }
 
-static void * __init __alloc_bootmem_core(struct bootmem_data *bdata,
-                               unsigned long size, unsigned long align,
-                               unsigned long goal, unsigned long limit)
+static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
+                                       unsigned long size, unsigned long align,
+                                       unsigned long goal, unsigned long limit)
 {
        unsigned long fallback = 0;
        unsigned long min, max, start, sidx, midx, step;
@@ -526,6 +521,11 @@ find_block:
                region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
                                start_off);
                memset(region, 0, size);
+               /*
+                * The min_count is set to 0 so that bootmem allocated blocks
+                * are never reported as leaks.
+                */
+               kmemleak_alloc(region, size, 0, 0);
                return region;
        }
 
@@ -538,17 +538,41 @@ find_block:
        return NULL;
 }
 
+static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
+                                       unsigned long size, unsigned long align,
+                                       unsigned long goal, unsigned long limit)
+{
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc(size, GFP_NOWAIT);
+
+#ifdef CONFIG_HAVE_ARCH_BOOTMEM
+       {
+               bootmem_data_t *p_bdata;
+
+               p_bdata = bootmem_arch_preferred_node(bdata, size, align,
+                                                       goal, limit);
+               if (p_bdata)
+                       return alloc_bootmem_core(p_bdata, size, align,
+                                                       goal, limit);
+       }
+#endif
+       return NULL;
+}
+
 static void * __init ___alloc_bootmem_nopanic(unsigned long size,
                                        unsigned long align,
                                        unsigned long goal,
                                        unsigned long limit)
 {
        bootmem_data_t *bdata;
+       void *region;
 
 restart:
-       list_for_each_entry(bdata, &bdata_list, list) {
-               void *region;
+       region = alloc_arch_preferred_bootmem(NULL, size, align, goal, limit);
+       if (region)
+               return region;
 
+       list_for_each_entry(bdata, &bdata_list, list) {
                if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
                        continue;
                if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))
@@ -626,6 +650,10 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
 {
        void *ptr;
 
+       ptr = alloc_arch_preferred_bootmem(bdata, size, align, goal, limit);
+       if (ptr)
+               return ptr;
+
        ptr = alloc_bootmem_core(bdata, size, align, goal, limit);
        if (ptr)
                return ptr;
@@ -651,6 +679,9 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
 void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
                                   unsigned long align, unsigned long goal)
 {
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
        return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
 }
 
@@ -682,6 +713,13 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
 {
        void *ptr;
 
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
+       ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
+       if (ptr)
+               return ptr;
+
        ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
        if (ptr)
                return ptr;
@@ -730,6 +768,9 @@ void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
 void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
                                       unsigned long align, unsigned long goal)
 {
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
        return ___alloc_bootmem_node(pgdat->bdata, size, align,
                                goal, ARCH_LOW_ADDRESS_LIMIT);
 }