#include <linux/pfn.h>
#include <linux/bootmem.h>
#include <linux/module.h>
+#include <linux/kmemleak.h>
#include <asm/bug.h>
#include <asm/io.h>
static int bootmem_debug;
-/*
- * If an arch needs to apply workarounds to bootmem allocation, it can
- * set CONFIG_HAVE_ARCH_BOOTMEM and define a wrapper around
- * __alloc_bootmem_core().
- */
-#ifndef CONFIG_HAVE_ARCH_BOOTMEM
-#define alloc_bootmem_core(bdata, size, align, goal, limit) \
- __alloc_bootmem_core((bdata), (size), (align), (goal), (limit))
-#endif
-
static int __init bootmem_debug_setup(char *buf)
{
bootmem_debug = 1;
return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
}
+/*
+ * free_bootmem_late - free bootmem pages directly to page allocator
+ * @addr: starting address of the range
+ * @size: size of the range in bytes
+ *
+ * This is only useful when the bootmem allocator has already been torn
+ * down, but we are still initializing the system. Pages are given directly
+ * to the page allocator, no bootmem metadata is updated because it is gone.
+ */
+void __init free_bootmem_late(unsigned long addr, unsigned long size)
+{
+ unsigned long cursor, end;
+
+ kmemleak_free_part(__va(addr), size);
+
+ cursor = PFN_UP(addr);
+ end = PFN_DOWN(addr + size);
+
+ for (; cursor < end; cursor++) {
+ __free_pages_bootmem(pfn_to_page(cursor), 0);
+ totalram_pages++;
+ }
+}
+
static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
{
int aligned;
{
unsigned long start, end;
+ kmemleak_free_part(__va(physaddr), size);
+
start = PFN_UP(physaddr);
end = PFN_DOWN(physaddr + size);
{
unsigned long start, end;
+ kmemleak_free_part(__va(addr), size);
+
start = PFN_UP(addr);
end = PFN_DOWN(addr + size);
return ALIGN(base + off, align) - base;
}
-static void * __init __alloc_bootmem_core(struct bootmem_data *bdata,
- unsigned long size, unsigned long align,
- unsigned long goal, unsigned long limit)
+static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
+ unsigned long size, unsigned long align,
+ unsigned long goal, unsigned long limit)
{
unsigned long fallback = 0;
unsigned long min, max, start, sidx, midx, step;
region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
start_off);
memset(region, 0, size);
+ /*
+ * The min_count is set to 0 so that bootmem allocated blocks
+ * are never reported as leaks.
+ */
+ kmemleak_alloc(region, size, 0, 0);
return region;
}
return NULL;
}
+static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
+ unsigned long size, unsigned long align,
+ unsigned long goal, unsigned long limit)
+{
+ if (WARN_ON_ONCE(slab_is_available()))
+ return kzalloc(size, GFP_NOWAIT);
+
+#ifdef CONFIG_HAVE_ARCH_BOOTMEM
+ {
+ bootmem_data_t *p_bdata;
+
+ p_bdata = bootmem_arch_preferred_node(bdata, size, align,
+ goal, limit);
+ if (p_bdata)
+ return alloc_bootmem_core(p_bdata, size, align,
+ goal, limit);
+ }
+#endif
+ return NULL;
+}
+
static void * __init ___alloc_bootmem_nopanic(unsigned long size,
unsigned long align,
unsigned long goal,
unsigned long limit)
{
bootmem_data_t *bdata;
+ void *region;
restart:
- list_for_each_entry(bdata, &bdata_list, list) {
- void *region;
+ region = alloc_arch_preferred_bootmem(NULL, size, align, goal, limit);
+ if (region)
+ return region;
+ list_for_each_entry(bdata, &bdata_list, list) {
if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
continue;
if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))
{
void *ptr;
+ ptr = alloc_arch_preferred_bootmem(bdata, size, align, goal, limit);
+ if (ptr)
+ return ptr;
+
ptr = alloc_bootmem_core(bdata, size, align, goal, limit);
if (ptr)
return ptr;
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
unsigned long align, unsigned long goal)
{
+ if (WARN_ON_ONCE(slab_is_available()))
+ return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
}
{
void *ptr;
+ if (WARN_ON_ONCE(slab_is_available()))
+ return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
+ ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
+ if (ptr)
+ return ptr;
+
ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
if (ptr)
return ptr;
void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
unsigned long align, unsigned long goal)
{
+ if (WARN_ON_ONCE(slab_is_available()))
+ return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
return ___alloc_bootmem_node(pgdat->bdata, size, align,
goal, ARCH_LOW_ADDRESS_LIMIT);
}