*
* Overloading of page flags that are otherwise used for LRU management.
*
- * PageActive The slab is used as a cpu cache. Allocations
- * may be performed from the slab. The slab is not
- * on any slab list and cannot be moved onto one.
- * The cpu slab may be equipped with an additioanl
+ * PageActive The slab is frozen and exempt from list processing.
+ * This means that the slab is dedicated to a purpose
+ * such as satisfying allocations for a specific
+ * processor. Objects may be freed in the slab while
+ * it is frozen but slab_free will then skip the usual
+ * list operations. It is up to the processor holding
+ * the slab to integrate the slab into the slab lists
+ * when the slab is no longer needed.
+ *
+ * One use of this flag is to mark slabs that are
+ * used for allocations. Then such a slab becomes a cpu
+ * slab. The cpu slab may be equipped with an additional
* lockless_freelist that allows lockless access to
* free objects in addition to the regular freelist
* that requires the slab lock.
* the fast path and disables lockless freelists.
*/
-static inline int SlabDebug(struct page *page)
-{
+#define FROZEN (1 << PG_active)
+
#ifdef CONFIG_SLUB_DEBUG
- return PageError(page);
+#define SLABDEBUG (1 << PG_error)
#else
- return 0;
+#define SLABDEBUG 0
#endif
+
+static inline int SlabFrozen(struct page *page)
+{
+ return page->flags & FROZEN;
+}
+
+static inline void SetSlabFrozen(struct page *page)
+{
+ page->flags |= FROZEN;
+}
+
+static inline void ClearSlabFrozen(struct page *page)
+{
+ page->flags &= ~FROZEN;
+}
+
+static inline int SlabDebug(struct page *page)
+{
+ return page->flags & SLABDEBUG;
}
static inline void SetSlabDebug(struct page *page)
{
-#ifdef CONFIG_SLUB_DEBUG
- SetPageError(page);
-#endif
+ page->flags |= SLABDEBUG;
}
static inline void ClearSlabDebug(struct page *page)
{
-#ifdef CONFIG_SLUB_DEBUG
- ClearPageError(page);
-#endif
+ page->flags &= ~SLABDEBUG;
}
/*
* On 32 bit platforms the limit is 256k. On 64bit platforms
* the limit is 512k.
*
- * Debugging or ctor/dtors may create a need to move the free
+ * Debugging or ctor may create a need to move the free
* pointer. Fail if this happens.
*/
if (s->size >= 65535 * sizeof(void *)) {
BUG_ON(s->flags & (SLAB_RED_ZONE | SLAB_POISON |
SLAB_STORE_USER | SLAB_DESTROY_BY_RCU));
- BUG_ON(s->ctor || s->dtor);
+ BUG_ON(s->ctor);
}
else
/*
}
if (unlikely(s->ctor))
- s->ctor(object, s, SLAB_CTOR_CONSTRUCTOR);
+ s->ctor(object, s, 0);
}
static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
{
int pages = 1 << s->order;
- if (unlikely(SlabDebug(page) || s->dtor)) {
+ if (unlikely(SlabDebug(page))) {
void *p;
slab_pad_check(s, page);
- for_each_object(p, s, page_address(page)) {
- if (s->dtor)
- s->dtor(p, s, 0);
+ for_each_object(p, s, page_address(page))
check_object(s, page, p, 0);
- }
}
mod_zone_page_state(page_zone(page),
*
* Must hold list_lock.
*/
-static int lock_and_del_slab(struct kmem_cache_node *n, struct page *page)
+static inline int lock_and_freeze_slab(struct kmem_cache_node *n, struct page *page)
{
if (slab_trylock(page)) {
list_del(&page->lru);
n->nr_partial--;
+ SetSlabFrozen(page);
return 1;
}
return 0;
spin_lock(&n->list_lock);
list_for_each_entry(page, &n->partial, lru)
- if (lock_and_del_slab(n, page))
+ if (lock_and_freeze_slab(n, page))
goto out;
page = NULL;
out:
*
* On exit the slab lock will have been dropped.
*/
-static void putback_slab(struct kmem_cache *s, struct page *page)
+static void unfreeze_slab(struct kmem_cache *s, struct page *page)
{
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+ ClearSlabFrozen(page);
if (page->inuse) {
if (page->freelist)
page->inuse--;
}
s->cpu_slab[cpu] = NULL;
- ClearPageActive(page);
-
- putback_slab(s, page);
+ unfreeze_slab(s, page);
}
static void flush_slab(struct kmem_cache *s, struct page *page, int cpu)
new_slab:
page = get_partial(s, gfpflags, node);
if (page) {
-have_slab:
s->cpu_slab[cpu] = page;
- SetPageActive(page);
goto load_freelist;
}
flush_slab(s, s->cpu_slab[cpu], cpu);
}
slab_lock(page);
- goto have_slab;
+ SetSlabFrozen(page);
+ s->cpu_slab[cpu] = page;
+ goto load_freelist;
}
return NULL;
debug:
page->freelist = object;
page->inuse--;
- if (unlikely(PageActive(page)))
- /*
- * Cpu slabs are never on partial lists and are
- * never freed.
- */
+ if (unlikely(SlabFrozen(page)))
goto out_unlock;
if (unlikely(!page->inuse))
debug:
if (!free_object_checks(s, page, x))
goto out_unlock;
- if (!PageActive(page) && !page->freelist)
+ if (!SlabFrozen(page) && !page->freelist)
remove_full(s, page);
if (s->flags & SLAB_STORE_USER)
set_track(s, x, TRACK_FREE, addr);
* then we should never poison the object itself.
*/
if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
- !s->ctor && !s->dtor)
+ !s->ctor)
s->flags |= __OBJECT_POISON;
else
s->flags &= ~__OBJECT_POISON;
#ifdef CONFIG_SLUB_DEBUG
if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
- s->ctor || s->dtor)) {
+ s->ctor)) {
/*
* Relocate free pointer after the object if it is not
* permitted to overwrite the first word of the object on
static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
const char *name, size_t size,
size_t align, unsigned long flags,
- void (*ctor)(void *, struct kmem_cache *, unsigned long),
- void (*dtor)(void *, struct kmem_cache *, unsigned long))
+ void (*ctor)(void *, struct kmem_cache *, unsigned long))
{
memset(s, 0, kmem_size);
s->name = name;
s->ctor = ctor;
- s->dtor = dtor;
s->objsize = size;
s->flags = flags;
s->align = align;
down_write(&slub_lock);
if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
- flags, NULL, NULL))
+ flags, NULL))
goto panic;
list_add(&s->list, &slab_caches);
if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
return 1;
- if (s->ctor || s->dtor)
+ if (s->ctor)
return 1;
return 0;
static struct kmem_cache *find_mergeable(size_t size,
size_t align, unsigned long flags,
- void (*ctor)(void *, struct kmem_cache *, unsigned long),
- void (*dtor)(void *, struct kmem_cache *, unsigned long))
+ void (*ctor)(void *, struct kmem_cache *, unsigned long))
{
struct list_head *h;
if (slub_nomerge || (flags & SLUB_NEVER_MERGE))
return NULL;
- if (ctor || dtor)
+ if (ctor)
return NULL;
size = ALIGN(size, sizeof(void *));
{
struct kmem_cache *s;
+ BUG_ON(dtor);
down_write(&slub_lock);
- s = find_mergeable(size, align, flags, dtor, ctor);
+ s = find_mergeable(size, align, flags, ctor);
if (s) {
s->refcount++;
/*
} else {
s = kmalloc(kmem_size, GFP_KERNEL);
if (s && kmem_cache_open(s, GFP_KERNEL, name,
- size, align, flags, ctor, dtor)) {
+ size, align, flags, ctor)) {
if (sysfs_slab_add(s)) {
kfree(s);
goto err;
}
SLAB_ATTR_RO(ctor);
-static ssize_t dtor_show(struct kmem_cache *s, char *buf)
-{
- if (s->dtor) {
- int n = sprint_symbol(buf, (unsigned long)s->dtor);
-
- return n + sprintf(buf + n, "\n");
- }
- return 0;
-}
-SLAB_ATTR_RO(dtor);
-
static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
return sprintf(buf, "%d\n", s->refcount - 1);
&partial_attr.attr,
&cpu_slabs_attr.attr,
&ctor_attr.attr,
- &dtor_attr.attr,
&aliases_attr.attr,
&align_attr.attr,
&sanity_checks_attr.attr,
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff