writeback: balance_dirty_pages() shall write more than dirtied pages

[safe/jmp/linux-2.6] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index 6a1ad0b..7dfa481 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -673,6 +673,7 @@ static enum {
        NONE,
        PARTIAL_AC,
        PARTIAL_L3,
+       EARLY,
        FULL
 } g_cpucache_up;
 
@@ -681,7 +682,7 @@ static enum {
  */
 int slab_is_available(void)
 {
-       return g_cpucache_up == FULL;
+       return g_cpucache_up >= EARLY;
 }
 
 static DEFINE_PER_CPU(struct delayed_work, reap_work);
@@ -811,7 +812,6 @@ static void __slab_error(const char *function, struct kmem_cache *cachep,
   */
 
 static int use_alien_caches __read_mostly = 1;
-static int numa_platform __read_mostly = 1;
 static int __init noaliencache_setup(char *s)
 {
        use_alien_caches = 0;
@@ -1370,10 +1370,8 @@ void __init kmem_cache_init(void)
        int order;
        int node;
 
-       if (num_possible_nodes() == 1) {
+       if (num_possible_nodes() == 1)
                use_alien_caches = 0;
-               numa_platform = 0;
-       }
 
        for (i = 0; i < NUM_INIT_LISTS; i++) {
                kmem_list3_init(&initkmem_list3[i]);
@@ -1386,7 +1384,7 @@ void __init kmem_cache_init(void)
         * Fragmentation resistance on low memory - only use bigger
         * page orders on machines with more than 32MB of memory.
         */
-       if (num_physpages > (32 << 20) >> PAGE_SHIFT)
+       if (totalram_pages > (32 << 20) >> PAGE_SHIFT)
                slab_break_gfp_order = BREAK_GFP_ORDER_HI;
 
        /* Bootstrap is tricky, because several objects are allocated
@@ -1545,23 +1543,26 @@ void __init kmem_cache_init(void)
                }
        }
 
-       /* 6) resize the head arrays to their final sizes */
-       {
-               struct kmem_cache *cachep;
-               mutex_lock(&cache_chain_mutex);
-               list_for_each_entry(cachep, &cache_chain, next)
-                       if (enable_cpucache(cachep, GFP_NOWAIT))
-                               BUG();
-               mutex_unlock(&cache_chain_mutex);
-       }
+       g_cpucache_up = EARLY;
+}
 
-       /* Annotate slab for lockdep -- annotate the malloc caches */
-       init_lock_keys();
+void __init kmem_cache_init_late(void)
+{
+       struct kmem_cache *cachep;
 
+       /* 6) resize the head arrays to their final sizes */
+       mutex_lock(&cache_chain_mutex);
+       list_for_each_entry(cachep, &cache_chain, next)
+               if (enable_cpucache(cachep, GFP_NOWAIT))
+                       BUG();
+       mutex_unlock(&cache_chain_mutex);
 
        /* Done! */
        g_cpucache_up = FULL;
 
+       /* Annotate slab for lockdep -- annotate the malloc caches */
+       init_lock_keys();
+
        /*
         * Register a cpu startup notifier callback that initializes
         * cpu_cache_get for all new cpus
@@ -1612,7 +1613,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
        if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                flags |= __GFP_RECLAIMABLE;
 
-       page = alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
+       page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
        if (!page)
                return NULL;
 
@@ -2034,7 +2035,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
                        for_each_online_node(node) {
                                cachep->nodelists[node] =
                                    kmalloc_node(sizeof(struct kmem_list3),
-                                               GFP_KERNEL, node);
+                                               gfp, node);
                                BUG_ON(!cachep->nodelists[node]);
                                kmem_list3_init(cachep->nodelists[node]);
                        }
@@ -2296,6 +2297,15 @@ kmem_cache_create (const char *name, size_t size, size_t align,
                /* really off slab. No need for manual alignment */
                slab_size =
                    cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
+
+#ifdef CONFIG_PAGE_POISONING
+               /* If we're going to use the generic kernel_map_pages()
+                * poisoning, then it's going to smash the contents of
+                * the redzone and userword anyhow, so switch them off.
+                */
+               if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
+                       flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
+#endif
        }
 
        cachep->colour_off = cache_line_size();
@@ -2537,7 +2547,7 @@ void kmem_cache_destroy(struct kmem_cache *cachep)
        }
 
        if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
-               synchronize_rcu();
+               rcu_barrier();
 
        __kmem_cache_destroy(cachep);
        mutex_unlock(&cache_chain_mutex);
@@ -3178,7 +3188,7 @@ retry:
                if (local_flags & __GFP_WAIT)
                        local_irq_enable();
                kmem_flagcheck(cache, flags);
-               obj = kmem_getpages(cache, local_flags, -1);
+               obj = kmem_getpages(cache, local_flags, numa_node_id());
                if (local_flags & __GFP_WAIT)
                        local_irq_disable();
                if (obj) {
@@ -3286,6 +3296,8 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
        unsigned long save_flags;
        void *ptr;
 
+       flags &= gfp_allowed_mask;
+
        lockdep_trace_alloc(flags);
 
        if (slab_should_failslab(cachep, flags))
@@ -3369,6 +3381,8 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
        unsigned long save_flags;
        void *objp;
 
+       flags &= gfp_allowed_mask;
+
        lockdep_trace_alloc(flags);
 
        if (slab_should_failslab(cachep, flags))
@@ -3511,7 +3525,7 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
         * variable to skip the call, which is mostly likely to be present in
         * the cache.
         */
-       if (numa_platform && cache_free_alien(cachep, objp))
+       if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
                return;
 
        if (likely(ac->avail < ac->limit)) {