percpu: add optimized generic percpu accessors

[safe/jmp/linux-2.6] / arch / x86 / xen / mmu.c

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 49697d8..7bc7852 100644 (file)
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -154,13 +154,13 @@ void xen_setup_mfn_list_list(void)
 {
        unsigned pfn, idx;
 
-       for(pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn += P2M_ENTRIES_PER_PAGE) {
+       for (pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn += P2M_ENTRIES_PER_PAGE) {
                unsigned topidx = p2m_top_index(pfn);
 
                p2m_top_mfn[topidx] = virt_to_mfn(p2m_top[topidx]);
        }
 
-       for(idx = 0; idx < ARRAY_SIZE(p2m_top_mfn_list); idx++) {
+       for (idx = 0; idx < ARRAY_SIZE(p2m_top_mfn_list); idx++) {
                unsigned topidx = idx * P2M_ENTRIES_PER_PAGE;
                p2m_top_mfn_list[idx] = virt_to_mfn(&p2m_top_mfn[topidx]);
        }
@@ -179,7 +179,7 @@ void __init xen_build_dynamic_phys_to_machine(void)
        unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
        unsigned pfn;
 
-       for(pfn = 0; pfn < max_pfn; pfn += P2M_ENTRIES_PER_PAGE) {
+       for (pfn = 0; pfn < max_pfn; pfn += P2M_ENTRIES_PER_PAGE) {
                unsigned topidx = p2m_top_index(pfn);
 
                p2m_top[topidx] = &mfn_list[pfn];
@@ -207,7 +207,7 @@ static void alloc_p2m(unsigned long **pp, unsigned long *mfnp)
        p = (void *)__get_free_page(GFP_KERNEL | __GFP_NOFAIL);
        BUG_ON(p == NULL);
 
-       for(i = 0; i < P2M_ENTRIES_PER_PAGE; i++)
+       for (i = 0; i < P2M_ENTRIES_PER_PAGE; i++)
                p[i] = INVALID_P2M_ENTRY;
 
        if (cmpxchg(pp, p2m_missing, p) != p2m_missing)
@@ -407,7 +407,8 @@ out:
                preempt_enable();
 }
 
-pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
+                                unsigned long addr, pte_t *ptep)
 {
        /* Just return the pte as-is.  We preserve the bits on commit */
        return *ptep;
@@ -661,12 +662,11 @@ void xen_set_pgd(pgd_t *ptr, pgd_t val)
  * For 64-bit, we must skip the Xen hole in the middle of the address
  * space, just after the big x86-64 virtual hole.
  */
-static int xen_pgd_walk(struct mm_struct *mm,
-                       int (*func)(struct mm_struct *mm, struct page *,
-                                   enum pt_level),
-                       unsigned long limit)
+static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
+                         int (*func)(struct mm_struct *mm, struct page *,
+                                     enum pt_level),
+                         unsigned long limit)
 {
-       pgd_t *pgd = mm->pgd;
        int flush = 0;
        unsigned hole_low, hole_high;
        unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
@@ -753,6 +753,14 @@ out:
        return flush;
 }
 
+static int xen_pgd_walk(struct mm_struct *mm,
+                       int (*func)(struct mm_struct *mm, struct page *,
+                                   enum pt_level),
+                       unsigned long limit)
+{
+       return __xen_pgd_walk(mm, mm->pgd, func, limit);
+}
+
 /* If we're using split pte locks, then take the page's lock and
    return a pointer to it.  Otherwise return NULL. */
 static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
@@ -850,13 +858,16 @@ static int xen_pin_page(struct mm_struct *mm, struct page *page,
    read-only, and can be pinned. */
 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
 {
+       vm_unmap_aliases();
+
        xen_mc_batch();
 
-       if (xen_pgd_walk(mm, xen_pin_page, USER_LIMIT)) {
-               /* re-enable interrupts for kmap_flush_unused */
+       if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
+               /* re-enable interrupts for flushing */
                xen_mc_issue(0);
+
                kmap_flush_unused();
-               vm_unmap_aliases();
+
                xen_mc_batch();
        }
 
@@ -868,7 +879,8 @@ static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
 
                if (user_pgd) {
                        xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
-                       xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(user_pgd)));
+                       xen_do_pin(MMUEXT_PIN_L4_TABLE,
+                                  PFN_DOWN(__pa(user_pgd)));
                }
        }
 #else /* CONFIG_X86_32 */
@@ -983,7 +995,8 @@ static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
                pgd_t *user_pgd = xen_get_user_pgd(pgd);
 
                if (user_pgd) {
-                       xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(user_pgd)));
+                       xen_do_pin(MMUEXT_UNPIN_TABLE,
+                                  PFN_DOWN(__pa(user_pgd)));
                        xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
                }
        }
@@ -995,7 +1008,7 @@ static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
                       PT_PMD);
 #endif
 
-       xen_pgd_walk(mm, xen_unpin_page, USER_LIMIT);
+       __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
 
        xen_mc_issue(0);
 }
@@ -1061,7 +1074,7 @@ static void drop_other_mm_ref(void *info)
 
        /* If this cpu still has a stale cr3 reference, then make sure
           it has been flushed. */
-       if (x86_read_percpu(xen_current_cr3) == __pa(mm->pgd)) {
+       if (percpu_read(xen_current_cr3) == __pa(mm->pgd)) {
                load_cr3(swapper_pg_dir);
                arch_flush_lazy_cpu_mode();
        }
@@ -1069,7 +1082,7 @@ static void drop_other_mm_ref(void *info)
 
 static void xen_drop_mm_ref(struct mm_struct *mm)
 {
-       cpumask_t mask;
+       cpumask_var_t mask;
        unsigned cpu;
 
        if (current->active_mm == mm) {
@@ -1081,7 +1094,16 @@ static void xen_drop_mm_ref(struct mm_struct *mm)
        }
 
        /* Get the "official" set of cpus referring to our pagetable. */
-       mask = mm->cpu_vm_mask;
+       if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
+               for_each_online_cpu(cpu) {
+                       if (!cpumask_test_cpu(cpu, &mm->cpu_vm_mask)
+                           && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
+                               continue;
+                       smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
+               }
+               return;
+       }
+       cpumask_copy(mask, &mm->cpu_vm_mask);
 
        /* It's possible that a vcpu may have a stale reference to our
           cr3, because its in lazy mode, and it hasn't yet flushed
@@ -1090,11 +1112,12 @@ static void xen_drop_mm_ref(struct mm_struct *mm)
           if needed. */
        for_each_online_cpu(cpu) {
                if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
-                       cpu_set(cpu, mask);
+                       cpumask_set_cpu(cpu, mask);
        }
 
-       if (!cpus_empty(mask))
-               smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
+       if (!cpumask_empty(mask))
+               smp_call_function_many(mask, drop_other_mm_ref, mm, 1);
+       free_cpumask_var(mask);
 }
 #else
 static void xen_drop_mm_ref(struct mm_struct *mm)