[PATCH] mm: slab: eliminate lock_cpu_hotplug from slab

[safe/jmp/linux-2.6] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index a65bc5e..3318252 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -313,7 +313,7 @@ static int drain_freelist(struct kmem_cache *cache,
 static void free_block(struct kmem_cache *cachep, void **objpp, int len,
                        int node);
 static int enable_cpucache(struct kmem_cache *cachep);
-static void cache_reap(void *unused);
+static void cache_reap(struct work_struct *unused);
 
 /*
  * This function must be completely optimized away if a constant is passed to
@@ -730,7 +730,10 @@ static inline void init_lock_keys(void)
 }
 #endif
 
-/* Guard access to the cache-chain. */
+/*
+ * 1. Guard access to the cache-chain.
+ * 2. Protect sanity of cpu_online_map against cpu hotplug events
+ */
 static DEFINE_MUTEX(cache_chain_mutex);
 static struct list_head cache_chain;
 
@@ -925,7 +928,7 @@ static void __devinit start_cpu_timer(int cpu)
         */
        if (keventd_up() && reap_work->work.func == NULL) {
                init_reap_node(cpu);
-               INIT_DELAYED_WORK(reap_work, cache_reap, NULL);
+               INIT_DELAYED_WORK(reap_work, cache_reap);
                schedule_delayed_work_on(cpu, reap_work, HZ + 3 * cpu);
        }
 }
@@ -1230,12 +1233,18 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb,
                        kfree(shared);
                        free_alien_cache(alien);
                }
-               mutex_unlock(&cache_chain_mutex);
                break;
        case CPU_ONLINE:
+               mutex_unlock(&cache_chain_mutex);
                start_cpu_timer(cpu);
                break;
 #ifdef CONFIG_HOTPLUG_CPU
+       case CPU_DOWN_PREPARE:
+               mutex_lock(&cache_chain_mutex);
+               break;
+       case CPU_DOWN_FAILED:
+               mutex_unlock(&cache_chain_mutex);
+               break;
        case CPU_DEAD:
                /*
                 * Even if all the cpus of a node are down, we don't free the
@@ -1246,8 +1255,8 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb,
                 * gets destroyed at kmem_cache_destroy().
                 */
                /* fall thru */
+#endif
        case CPU_UP_CANCELED:
-               mutex_lock(&cache_chain_mutex);
                list_for_each_entry(cachep, &cache_chain, next) {
                        struct array_cache *nc;
                        struct array_cache *shared;
@@ -1308,11 +1317,9 @@ free_array_cache:
                }
                mutex_unlock(&cache_chain_mutex);
                break;
-#endif
        }
        return NOTIFY_OK;
 bad:
-       mutex_unlock(&cache_chain_mutex);
        return NOTIFY_BAD;
 }
 
@@ -2098,11 +2105,9 @@ kmem_cache_create (const char *name, size_t size, size_t align,
        }
 
        /*
-        * Prevent CPUs from coming and going.
-        * lock_cpu_hotplug() nests outside cache_chain_mutex
+        * We use cache_chain_mutex to ensure a consistent view of
+        * cpu_online_map as well.  Please see cpuup_callback
         */
-       lock_cpu_hotplug();
-
        mutex_lock(&cache_chain_mutex);
 
        list_for_each_entry(pc, &cache_chain, next) {
@@ -2197,18 +2202,17 @@ kmem_cache_create (const char *name, size_t size, size_t align,
        if (flags & SLAB_RED_ZONE || flags & SLAB_STORE_USER)
                ralign = BYTES_PER_WORD;
 
-       /* 2) arch mandated alignment: disables debug if necessary */
+       /* 2) arch mandated alignment */
        if (ralign < ARCH_SLAB_MINALIGN) {
                ralign = ARCH_SLAB_MINALIGN;
-               if (ralign > BYTES_PER_WORD)
-                       flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
        }
-       /* 3) caller mandated alignment: disables debug if necessary */
+       /* 3) caller mandated alignment */
        if (ralign < align) {
                ralign = align;
-               if (ralign > BYTES_PER_WORD)
-                       flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
        }
+       /* disable debug if necessary */
+       if (ralign > BYTES_PER_WORD)
+               flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
        /*
         * 4) Store it.
         */
@@ -2326,7 +2330,6 @@ oops:
                panic("kmem_cache_create(): failed to create slab `%s'\n",
                      name);
        mutex_unlock(&cache_chain_mutex);
-       unlock_cpu_hotplug();
        return cachep;
 }
 EXPORT_SYMBOL(kmem_cache_create);
@@ -2444,6 +2447,7 @@ out:
        return nr_freed;
 }
 
+/* Called with cache_chain_mutex held to protect against cpu hotplug */
 static int __cache_shrink(struct kmem_cache *cachep)
 {
        int ret = 0, i = 0;
@@ -2474,9 +2478,13 @@ static int __cache_shrink(struct kmem_cache *cachep)
  */
 int kmem_cache_shrink(struct kmem_cache *cachep)
 {
+       int ret;
        BUG_ON(!cachep || in_interrupt());
 
-       return __cache_shrink(cachep);
+       mutex_lock(&cache_chain_mutex);
+       ret = __cache_shrink(cachep);
+       mutex_unlock(&cache_chain_mutex);
+       return ret;
 }
 EXPORT_SYMBOL(kmem_cache_shrink);
 
@@ -2500,23 +2508,16 @@ void kmem_cache_destroy(struct kmem_cache *cachep)
 {
        BUG_ON(!cachep || in_interrupt());
 
-       /* Don't let CPUs to come and go */
-       lock_cpu_hotplug();
-
        /* Find the cache in the chain of caches. */
        mutex_lock(&cache_chain_mutex);
        /*
         * the chain is never empty, cache_cache is never destroyed
         */
        list_del(&cachep->next);
-       mutex_unlock(&cache_chain_mutex);
-
        if (__cache_shrink(cachep)) {
                slab_error(cachep, "Can't free all objects");
-               mutex_lock(&cache_chain_mutex);
                list_add(&cachep->next, &cache_chain);
                mutex_unlock(&cache_chain_mutex);
-               unlock_cpu_hotplug();
                return;
        }
 
@@ -2524,7 +2525,7 @@ void kmem_cache_destroy(struct kmem_cache *cachep)
                synchronize_rcu();
 
        __kmem_cache_destroy(cachep);
-       unlock_cpu_hotplug();
+       mutex_unlock(&cache_chain_mutex);
 }
 EXPORT_SYMBOL(kmem_cache_destroy);
 
@@ -3063,6 +3064,12 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
 
                cachep->ctor(objp, cachep, ctor_flags);
        }
+#if ARCH_SLAB_MINALIGN
+       if ((u32)objp & (ARCH_SLAB_MINALIGN-1)) {
+               printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
+                      objp, ARCH_SLAB_MINALIGN);
+       }
+#endif
        return objp;
 }
 #else
@@ -3815,7 +3822,7 @@ void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
  * If we cannot acquire the cache chain mutex then just give up - we'll try
  * again on the next iteration.
  */
-static void cache_reap(void *unused)
+static void cache_reap(struct work_struct *unused)
 {
        struct kmem_cache *searchp;
        struct kmem_list3 *l3;