padata: Use a timer to handle remaining objects in the reorder queues

[safe/jmp/linux-2.6] / mm / percpu.c

diff --git a/mm/percpu.c b/mm/percpu.c
index 5adfc26..6e09741 100644 (file)
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -46,8 +46,6 @@
  *
  * To use this allocator, arch code should do the followings.
  *
- * - drop CONFIG_HAVE_LEGACY_PER_CPU_AREA
- *
  * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
  *   regular address to percpu pointer and back if they need to be
  *   different from the default
@@ -74,6 +72,7 @@
 #include <asm/cacheflush.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
+#include <asm/io.h>
 
 #define PCPU_SLOT_BASE_SHIFT           5       /* 1-31 shares the same slot */
 #define PCPU_DFL_MAP_ALLOC             16      /* start a map with 16 ents */
@@ -81,13 +80,15 @@
 /* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
 #ifndef __addr_to_pcpu_ptr
 #define __addr_to_pcpu_ptr(addr)                                       \
-       (void *)((unsigned long)(addr) - (unsigned long)pcpu_base_addr  \
-                + (unsigned long)__per_cpu_start)
+       (void __percpu *)((unsigned long)(addr) -                       \
+                         (unsigned long)pcpu_base_addr +               \
+                         (unsigned long)__per_cpu_start)
 #endif
 #ifndef __pcpu_ptr_to_addr
 #define __pcpu_ptr_to_addr(ptr)                                                \
-       (void *)((unsigned long)(ptr) + (unsigned long)pcpu_base_addr   \
-                - (unsigned long)__per_cpu_start)
+       (void __force *)((unsigned long)(ptr) +                         \
+                        (unsigned long)pcpu_base_addr -                \
+                        (unsigned long)__per_cpu_start)
 #endif
 
 struct pcpu_chunk {
@@ -914,11 +915,10 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
        int rs, re;
 
        /* quick path, check whether it's empty already */
-       pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
-               if (rs == page_start && re == page_end)
-                       return;
-               break;
-       }
+       rs = page_start;
+       pcpu_next_unpop(chunk, &rs, &re, page_end);
+       if (rs == page_start && re == page_end)
+               return;
 
        /* immutable chunks can't be depopulated */
        WARN_ON(chunk->immutable);
@@ -969,11 +969,10 @@ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
        int rs, re, rc;
 
        /* quick path, check whether all pages are already there */
-       pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end) {
-               if (rs == page_start && re == page_end)
-                       goto clear;
-               break;
-       }
+       rs = page_start;
+       pcpu_next_pop(chunk, &rs, &re, page_end);
+       if (rs == page_start && re == page_end)
+               goto clear;
 
        /* need to allocate and map pages, this chunk can't be immutable */
        WARN_ON(chunk->immutable);
@@ -1068,7 +1067,7 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-static void *pcpu_alloc(size_t size, size_t align, bool reserved)
+static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
 {
        static int warn_limit = 10;
        struct pcpu_chunk *chunk;
@@ -1197,7 +1196,7 @@ fail_unlock_mutex:
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-void *__alloc_percpu(size_t size, size_t align)
+void __percpu *__alloc_percpu(size_t size, size_t align)
 {
        return pcpu_alloc(size, align, false);
 }
@@ -1218,7 +1217,7 @@ EXPORT_SYMBOL_GPL(__alloc_percpu);
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-void *__alloc_reserved_percpu(size_t size, size_t align)
+void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
 {
        return pcpu_alloc(size, align, true);
 }
@@ -1270,9 +1269,9 @@ static void pcpu_reclaim(struct work_struct *work)
  * CONTEXT:
  * Can be called from atomic context.
  */
-void free_percpu(void *ptr)
+void free_percpu(void __percpu *ptr)
 {
-       void *addr = __pcpu_ptr_to_addr(ptr);
+       void *addr;
        struct pcpu_chunk *chunk;
        unsigned long flags;
        int off;
@@ -1280,6 +1279,8 @@ void free_percpu(void *ptr)
        if (!ptr)
                return;
 
+       addr = __pcpu_ptr_to_addr(ptr);
+
        spin_lock_irqsave(&pcpu_lock, flags);
 
        chunk = pcpu_chunk_addr_search(addr);
@@ -1302,6 +1303,53 @@ void free_percpu(void *ptr)
 }
 EXPORT_SYMBOL_GPL(free_percpu);
 
+/**
+ * is_kernel_percpu_address - test whether address is from static percpu area
+ * @addr: address to test
+ *
+ * Test whether @addr belongs to in-kernel static percpu area.  Module
+ * static percpu areas are not considered.  For those, use
+ * is_module_percpu_address().
+ *
+ * RETURNS:
+ * %true if @addr is from in-kernel static percpu area, %false otherwise.
+ */
+bool is_kernel_percpu_address(unsigned long addr)
+{
+       const size_t static_size = __per_cpu_end - __per_cpu_start;
+       void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
+       unsigned int cpu;
+
+       for_each_possible_cpu(cpu) {
+               void *start = per_cpu_ptr(base, cpu);
+
+               if ((void *)addr >= start && (void *)addr < start + static_size)
+                       return true;
+        }
+       return false;
+}
+
+/**
+ * per_cpu_ptr_to_phys - convert translated percpu address to physical address
+ * @addr: the address to be converted to physical address
+ *
+ * Given @addr which is dereferenceable address obtained via one of
+ * percpu access macros, this function translates it into its physical
+ * address.  The caller is responsible for ensuring @addr stays valid
+ * until this function finishes.
+ *
+ * RETURNS:
+ * The physical address for @addr.
+ */
+phys_addr_t per_cpu_ptr_to_phys(void *addr)
+{
+       if ((unsigned long)addr < VMALLOC_START ||
+                       (unsigned long)addr >= VMALLOC_END)
+               return __pa(addr);
+       else
+               return page_to_phys(vmalloc_to_page(addr));
+}
+
 static inline size_t pcpu_calc_fc_sizes(size_t static_size,
                                        size_t reserved_size,
                                        ssize_t *dyn_sizep)