X-Git-Url: http://ftp.safe.ca/?a=blobdiff_plain;f=mm%2Fslub.c;h=e7ad123bb6a745be4ac1a62dd6f23152a3ff0536;hb=a35afb830f8d71ec211531aeb9a621b09a2efb39;hp=b39c8a69a4ff4f79faecc51b6d8cfeaabf707754;hpb=bcf889f96597137760c6edfdd0ee59fd37cb108c;p=safe%2Fjmp%2Flinux-2.6 diff --git a/mm/slub.c b/mm/slub.c index b39c8a6..e7ad123 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -78,10 +78,18 @@ * * Overloading of page flags that are otherwise used for LRU management. * - * PageActive The slab is used as a cpu cache. Allocations - * may be performed from the slab. The slab is not - * on any slab list and cannot be moved onto one. - * The cpu slab may be equipped with an additioanl + * PageActive The slab is frozen and exempt from list processing. + * This means that the slab is dedicated to a purpose + * such as satisfying allocations for a specific + * processor. Objects may be freed in the slab while + * it is frozen but slab_free will then skip the usual + * list operations. It is up to the processor holding + * the slab to integrate the slab into the slab lists + * when the slab is no longer needed. + * + * One use of this flag is to mark slabs that are + * used for allocations. Then such a slab becomes a cpu + * slab. The cpu slab may be equipped with an additional * lockless_freelist that allows lockless access to * free objects in addition to the regular freelist * that requires the slab lock. @@ -91,27 +99,42 @@ * the fast path and disables lockless freelists. */ -static inline int SlabDebug(struct page *page) -{ +#define FROZEN (1 << PG_active) + #ifdef CONFIG_SLUB_DEBUG - return PageError(page); +#define SLABDEBUG (1 << PG_error) #else - return 0; +#define SLABDEBUG 0 #endif + +static inline int SlabFrozen(struct page *page) +{ + return page->flags & FROZEN; +} + +static inline void SetSlabFrozen(struct page *page) +{ + page->flags |= FROZEN; +} + +static inline void ClearSlabFrozen(struct page *page) +{ + page->flags &= ~FROZEN; +} + +static inline int SlabDebug(struct page *page) +{ + return page->flags & SLABDEBUG; } static inline void SetSlabDebug(struct page *page) { -#ifdef CONFIG_SLUB_DEBUG - SetPageError(page); -#endif + page->flags |= SLABDEBUG; } static inline void ClearSlabDebug(struct page *page) { -#ifdef CONFIG_SLUB_DEBUG - ClearPageError(page); -#endif + page->flags &= ~SLABDEBUG; } /* @@ -891,13 +914,13 @@ static void kmem_cache_open_debug_check(struct kmem_cache *s) * On 32 bit platforms the limit is 256k. On 64bit platforms * the limit is 512k. * - * Debugging or ctor/dtors may create a need to move the free + * Debugging or ctor may create a need to move the free * pointer. Fail if this happens. */ if (s->size >= 65535 * sizeof(void *)) { BUG_ON(s->flags & (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | SLAB_DESTROY_BY_RCU)); - BUG_ON(s->ctor || s->dtor); + BUG_ON(s->ctor); } else /* @@ -971,7 +994,7 @@ static void setup_object(struct kmem_cache *s, struct page *page, } if (unlikely(s->ctor)) - s->ctor(object, s, SLAB_CTOR_CONSTRUCTOR); + s->ctor(object, s, 0); } static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) @@ -1030,15 +1053,12 @@ static void __free_slab(struct kmem_cache *s, struct page *page) { int pages = 1 << s->order; - if (unlikely(SlabDebug(page) || s->dtor)) { + if (unlikely(SlabDebug(page))) { void *p; slab_pad_check(s, page); - for_each_object(p, s, page_address(page)) { - if (s->dtor) - s->dtor(p, s, 0); + for_each_object(p, s, page_address(page)) check_object(s, page, p, 0); - } } mod_zone_page_state(page_zone(page), @@ -1138,11 +1158,12 @@ static void remove_partial(struct kmem_cache *s, * * Must hold list_lock. */ -static int lock_and_del_slab(struct kmem_cache_node *n, struct page *page) +static inline int lock_and_freeze_slab(struct kmem_cache_node *n, struct page *page) { if (slab_trylock(page)) { list_del(&page->lru); n->nr_partial--; + SetSlabFrozen(page); return 1; } return 0; @@ -1166,7 +1187,7 @@ static struct page *get_partial_node(struct kmem_cache_node *n) spin_lock(&n->list_lock); list_for_each_entry(page, &n->partial, lru) - if (lock_and_del_slab(n, page)) + if (lock_and_freeze_slab(n, page)) goto out; page = NULL; out: @@ -1245,10 +1266,11 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) * * On exit the slab lock will have been dropped. */ -static void putback_slab(struct kmem_cache *s, struct page *page) +static void unfreeze_slab(struct kmem_cache *s, struct page *page) { struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + ClearSlabFrozen(page); if (page->inuse) { if (page->freelist) @@ -1299,9 +1321,7 @@ static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu) page->inuse--; } s->cpu_slab[cpu] = NULL; - ClearPageActive(page); - - putback_slab(s, page); + unfreeze_slab(s, page); } static void flush_slab(struct kmem_cache *s, struct page *page, int cpu) @@ -1392,9 +1412,7 @@ another_slab: new_slab: page = get_partial(s, gfpflags, node); if (page) { -have_slab: s->cpu_slab[cpu] = page; - SetPageActive(page); goto load_freelist; } @@ -1424,7 +1442,9 @@ have_slab: flush_slab(s, s->cpu_slab[cpu], cpu); } slab_lock(page); - goto have_slab; + SetSlabFrozen(page); + s->cpu_slab[cpu] = page; + goto load_freelist; } return NULL; debug: @@ -1511,11 +1531,7 @@ checks_ok: page->freelist = object; page->inuse--; - if (unlikely(PageActive(page))) - /* - * Cpu slabs are never on partial lists and are - * never freed. - */ + if (unlikely(SlabFrozen(page))) goto out_unlock; if (unlikely(!page->inuse)) @@ -1547,7 +1563,7 @@ slab_empty: debug: if (!free_object_checks(s, page, x)) goto out_unlock; - if (!PageActive(page) && !page->freelist) + if (!SlabFrozen(page) && !page->freelist) remove_full(s, page); if (s->flags & SLAB_STORE_USER) set_track(s, x, TRACK_FREE, addr); @@ -1871,7 +1887,7 @@ static int calculate_sizes(struct kmem_cache *s) * then we should never poison the object itself. */ if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && - !s->ctor && !s->dtor) + !s->ctor) s->flags |= __OBJECT_POISON; else s->flags &= ~__OBJECT_POISON; @@ -1901,7 +1917,7 @@ static int calculate_sizes(struct kmem_cache *s) #ifdef CONFIG_SLUB_DEBUG if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || - s->ctor || s->dtor)) { + s->ctor)) { /* * Relocate free pointer after the object if it is not * permitted to overwrite the first word of the object on @@ -1970,13 +1986,11 @@ static int calculate_sizes(struct kmem_cache *s) static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags, const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + void (*ctor)(void *, struct kmem_cache *, unsigned long)) { memset(s, 0, kmem_size); s->name = name; s->ctor = ctor; - s->dtor = dtor; s->objsize = size; s->flags = flags; s->align = align; @@ -2161,7 +2175,7 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s, down_write(&slub_lock); if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN, - flags, NULL, NULL)) + flags, NULL)) goto panic; list_add(&s->list, &slab_caches); @@ -2463,7 +2477,7 @@ static int slab_unmergeable(struct kmem_cache *s) if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE)) return 1; - if (s->ctor || s->dtor) + if (s->ctor) return 1; return 0; @@ -2471,15 +2485,14 @@ static int slab_unmergeable(struct kmem_cache *s) static struct kmem_cache *find_mergeable(size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + void (*ctor)(void *, struct kmem_cache *, unsigned long)) { struct list_head *h; if (slub_nomerge || (flags & SLUB_NEVER_MERGE)) return NULL; - if (ctor || dtor) + if (ctor) return NULL; size = ALIGN(size, sizeof(void *)); @@ -2521,8 +2534,9 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, { struct kmem_cache *s; + BUG_ON(dtor); down_write(&slub_lock); - s = find_mergeable(size, align, flags, dtor, ctor); + s = find_mergeable(size, align, flags, ctor); if (s) { s->refcount++; /* @@ -2536,7 +2550,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, } else { s = kmalloc(kmem_size, GFP_KERNEL); if (s && kmem_cache_open(s, GFP_KERNEL, name, - size, align, flags, ctor, dtor)) { + size, align, flags, ctor)) { if (sysfs_slab_add(s)) { kfree(s); goto err; @@ -3177,17 +3191,6 @@ static ssize_t ctor_show(struct kmem_cache *s, char *buf) } SLAB_ATTR_RO(ctor); -static ssize_t dtor_show(struct kmem_cache *s, char *buf) -{ - if (s->dtor) { - int n = sprint_symbol(buf, (unsigned long)s->dtor); - - return n + sprintf(buf + n, "\n"); - } - return 0; -} -SLAB_ATTR_RO(dtor); - static ssize_t aliases_show(struct kmem_cache *s, char *buf) { return sprintf(buf, "%d\n", s->refcount - 1); @@ -3419,7 +3422,6 @@ static struct attribute * slab_attrs[] = { &partial_attr.attr, &cpu_slabs_attr.attr, &ctor_attr.attr, - &dtor_attr.attr, &aliases_attr.attr, &align_attr.attr, &sanity_checks_attr.attr,