#include <linux/rbtree.h>
#include <linux/radix-tree.h>
#include <linux/rcupdate.h>
-#include <linux/bootmem.h>
#include <linux/pfn.h>
+#include <linux/kmemleak.h>
#include <asm/atomic.h>
#include <asm/uaccess.h>
printk(KERN_WARNING
"vmap allocation for size %lu failed: "
"use vmalloc=<size> to increase size.\n", size);
+ kfree(va);
return ERR_PTR(-EBUSY);
}
DECLARE_BITMAP(alloc_map, VMAP_BBMAP_BITS);
DECLARE_BITMAP(dirty_map, VMAP_BBMAP_BITS);
union {
- struct {
- struct list_head free_list;
- struct list_head dirty_list;
- };
+ struct list_head free_list;
struct rcu_head rcu_head;
};
};
bitmap_zero(vb->alloc_map, VMAP_BBMAP_BITS);
bitmap_zero(vb->dirty_map, VMAP_BBMAP_BITS);
INIT_LIST_HEAD(&vb->free_list);
- INIT_LIST_HEAD(&vb->dirty_list);
vb_idx = addr_to_vb_idx(va->va_start);
spin_lock(&vmap_block_tree_lock);
struct vmap_block *tmp;
unsigned long vb_idx;
- spin_lock(&vb->vbq->lock);
- if (!list_empty(&vb->free_list))
- list_del(&vb->free_list);
- if (!list_empty(&vb->dirty_list))
- list_del(&vb->dirty_list);
- spin_unlock(&vb->vbq->lock);
+ BUG_ON(!list_empty(&vb->free_list));
vb_idx = addr_to_vb_idx(vb->va->va_start);
spin_lock(&vmap_block_tree_lock);
spin_lock(&vb->lock);
bitmap_allocate_region(vb->dirty_map, offset >> PAGE_SHIFT, order);
- if (!vb->dirty) {
- spin_lock(&vb->vbq->lock);
- list_add(&vb->dirty_list, &vb->vbq->dirty);
- spin_unlock(&vb->vbq->lock);
- }
+
vb->dirty += 1UL << order;
if (vb->dirty == VMAP_BBMAP_BITS) {
BUG_ON(vb->free || !list_empty(&vb->free_list));
/* Import existing vmlist entries. */
for (tmp = vmlist; tmp; tmp = tmp->next) {
- va = alloc_bootmem(sizeof(struct vmap_area));
+ va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
va->flags = tmp->flags | VM_VM_AREA;
va->va_start = (unsigned long)tmp->addr;
va->va_end = va->va_start + tmp->size;
void vfree(const void *addr)
{
BUG_ON(in_interrupt());
+
+ kmemleak_free(addr);
+
__vunmap(addr, 1);
}
EXPORT_SYMBOL(vfree);
void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
{
- return __vmalloc_area_node(area, gfp_mask, prot, -1,
- __builtin_return_address(0));
+ void *addr = __vmalloc_area_node(area, gfp_mask, prot, -1,
+ __builtin_return_address(0));
+
+ /*
+ * A ref_count = 3 is needed because the vm_struct and vmap_area
+ * structures allocated in the __get_vm_area_node() function contain
+ * references to the virtual address of the vmalloc'ed block.
+ */
+ kmemleak_alloc(addr, area->size - PAGE_SIZE, 3, gfp_mask);
+
+ return addr;
}
/**
int node, void *caller)
{
struct vm_struct *area;
+ void *addr;
+ unsigned long real_size = size;
size = PAGE_ALIGN(size);
if (!size || (size >> PAGE_SHIFT) > num_physpages)
if (!area)
return NULL;
- return __vmalloc_area_node(area, gfp_mask, prot, node, caller);
+ addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
+
+ /*
+ * A ref_count = 3 is needed because the vm_struct and vmap_area
+ * structures allocated in the __get_vm_area_node() function contain
+ * references to the virtual address of the vmalloc'ed block.
+ */
+ kmemleak_alloc(addr, real_size, 3, gfp_mask);
+
+ return addr;
}
void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)