[PATCH] mm: shrink_inactive_lis() nr_scan accounting fix

[safe/jmp/linux-2.6] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 791690d..7aa0181 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -55,7 +55,6 @@ unsigned long totalhigh_pages __read_mostly;
 long nr_swap_pages;
 int percpu_pagelist_fraction;
 
-static void fastcall free_hot_cold_page(struct page *page, int cold);
 static void __free_pages_ok(struct page *page, unsigned int order);
 
 /*
@@ -190,7 +189,7 @@ static void prep_compound_page(struct page *page, unsigned long order)
        for (i = 0; i < nr_pages; i++) {
                struct page *p = page + i;
 
-               SetPageCompound(p);
+               __SetPageCompound(p);
                set_page_private(p, (unsigned long)page);
        }
 }
@@ -209,7 +208,7 @@ static void destroy_compound_page(struct page *page, unsigned long order)
                if (unlikely(!PageCompound(p) |
                                (page_private(p) != (unsigned long)page)))
                        bad_page(page);
-               ClearPageCompound(p);
+               __ClearPageCompound(p);
        }
 }
 
@@ -448,28 +447,23 @@ void fastcall __init __free_pages_bootmem(struct page *page, unsigned int order)
        if (order == 0) {
                __ClearPageReserved(page);
                set_page_count(page, 0);
-
-               free_hot_cold_page(page, 0);
+               set_page_refs(page, 0);
+               __free_page(page);
        } else {
-               LIST_HEAD(list);
                int loop;
 
+               prefetchw(page);
                for (loop = 0; loop < BITS_PER_LONG; loop++) {
                        struct page *p = &page[loop];
 
-                       if (loop + 16 < BITS_PER_LONG)
-                               prefetchw(p + 16);
+                       if (loop + 1 < BITS_PER_LONG)
+                               prefetchw(p + 1);
                        __ClearPageReserved(p);
                        set_page_count(p, 0);
                }
 
-               arch_free_page(page, order);
-
-               mod_page_state(pgfree, 1 << order);
-
-               list_add(&page->lru, &list);
-               kernel_map_pages(page, 1 << order, 0);
-               free_pages_bulk(page_zone(page), 1, &list, order);
+               set_page_refs(page, order);
+               __free_pages(page, order);
        }
 }
 
@@ -590,21 +584,20 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
 }
 
 #ifdef CONFIG_NUMA
-/* Called from the slab reaper to drain remote pagesets */
-void drain_remote_pages(void)
+/*
+ * Called from the slab reaper to drain pagesets on a particular node that
+ * belong to the currently executing processor.
+ */
+void drain_node_pages(int nodeid)
 {
-       struct zone *zone;
-       int i;
+       int i, z;
        unsigned long flags;
 
        local_irq_save(flags);
-       for_each_zone(zone) {
+       for (z = 0; z < MAX_NR_ZONES; z++) {
+               struct zone *zone = NODE_DATA(nodeid)->node_zones + z;
                struct per_cpu_pageset *pset;
 
-               /* Do not drain local pagesets */
-               if (zone->zone_pgdat->node_id == numa_node_id())
-                       continue;
-
                pset = zone_pcp(zone, smp_processor_id());
                for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) {
                        struct per_cpu_pages *pcp;
@@ -753,6 +746,28 @@ static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
                clear_highpage(page + i);
 }
 
+#ifdef CONFIG_MMU
+/*
+ * split_page takes a non-compound higher-order page, and splits it into
+ * n (1<<order) sub-pages: page[0..n]
+ * Each sub-page must be freed individually.
+ *
+ * Note: this is probably too low level an operation for use in drivers.
+ * Please consult with lkml before using this in your driver.
+ */
+void split_page(struct page *page, unsigned int order)
+{
+       int i;
+
+       BUG_ON(PageCompound(page));
+       BUG_ON(!page_count(page));
+       for (i = 1; i < (1 << order); i++) {
+               BUG_ON(page_count(page + i));
+               set_page_count(page + i, 1);
+       }
+}
+#endif
+
 /*
  * Really, prep_compound_page() should be called from __rmqueue_bulk().  But
  * we cheat by calling it from here, in the order > 0 path.  Saves a branch
@@ -1215,24 +1230,22 @@ DEFINE_PER_CPU(long, nr_pagecache_local) = 0;
 
 static void __get_page_state(struct page_state *ret, int nr, cpumask_t *cpumask)
 {
-       int cpu = 0;
+       unsigned cpu;
 
        memset(ret, 0, nr * sizeof(unsigned long));
        cpus_and(*cpumask, *cpumask, cpu_online_map);
 
-       cpu = first_cpu(*cpumask);
-       while (cpu < NR_CPUS) {
-               unsigned long *in, *out, off;
-
-               if (!cpu_isset(cpu, *cpumask))
-                       continue;
+       for_each_cpu_mask(cpu, *cpumask) {
+               unsigned long *in;
+               unsigned long *out;
+               unsigned off;
+               unsigned next_cpu;
 
                in = (unsigned long *)&per_cpu(page_states, cpu);
 
-               cpu = next_cpu(cpu, *cpumask);
-
-               if (likely(cpu < NR_CPUS))
-                       prefetch(&per_cpu(page_states, cpu));
+               next_cpu = next_cpu(cpu, *cpumask);
+               if (likely(next_cpu < NR_CPUS))
+                       prefetch(&per_cpu(page_states, next_cpu));
 
                out = (unsigned long *)ret;
                for (off = 0; off < nr; off++)