x86: cpa: move flush to cpa

The set_memory_* and set_pages_* family of API's currently requires the
callers to do a global tlb flush after the function call; forgetting this is
a very nasty deathtrap. This patch moves the global tlb flush into
each of the callers

Signed-off-by: Arjan van de Ven <arjan@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index 8860c6e..4d5cc71 100644 (file)
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -572,7 +572,6 @@ static __init int init_k8_gatt(struct agp_kern_info *info)
                panic("Cannot allocate GATT table");
        if (set_memory_uc((unsigned long)gatt, gatt_size >> PAGE_SHIFT))
                panic("Could not set GART PTEs to uncacheable pages");
-       global_flush_tlb();
 
        memset(gatt, 0, gatt_size);
        agp_gatt_table = gatt;

diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index f7b941c..0d3369b 100644 (file)
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -752,15 +752,11 @@ void mark_rodata_ro(void)
                printk("Write protecting the kernel text: %luk\n", size >> 10);
 
 #ifdef CONFIG_CPA_DEBUG
-               global_flush_tlb();
-
                printk("Testing CPA: Reverting %lx-%lx\n", start, start+size);
                set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT);
-               global_flush_tlb();
 
                printk("Testing CPA: write protecting again\n");
                set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT);
-               global_flush_tlb();
 #endif
        }
 #endif
@@ -770,22 +766,12 @@ void mark_rodata_ro(void)
        printk("Write protecting the kernel read-only data: %luk\n",
               size >> 10);
 
-       /*
-        * set_pages_*() requires a global_flush_tlb() call after it.
-        * We do this after the printk so that if something went wrong in the
-        * change, the printk gets out at least to give a better debug hint
-        * of who is the culprit.
-        */
-       global_flush_tlb();
-
 #ifdef CONFIG_CPA_DEBUG
        printk("Testing CPA: undo %lx-%lx\n", start, start + size);
        set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
-       global_flush_tlb();
 
        printk("Testing CPA: write protecting again\n");
        set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
-       global_flush_tlb();
 #endif
 }
 #endif

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 4757be7..9b69fa5 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -610,22 +610,12 @@ void mark_rodata_ro(void)
        printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
               (end - start) >> 10);
 
-       /*
-        * set_memory_*() requires a global_flush_tlb() call after it.
-        * We do this after the printk so that if something went wrong in the
-        * change, the printk gets out at least to give a better debug hint
-        * of who is the culprit.
-        */
-       global_flush_tlb();
-
 #ifdef CONFIG_CPA_DEBUG
        printk("Testing CPA: undo %lx-%lx\n", start, end);
        set_memory_rw(start, (end-start) >> PAGE_SHIFT);
-       global_flush_tlb();
 
        printk("Testing CPA: again\n");
        set_memory_ro(start, (end-start) >> PAGE_SHIFT);
-       global_flush_tlb();
 #endif
 }
 #endif

diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index b86f66f..6a9a141 100644 (file)
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -96,8 +96,6 @@ static int ioremap_change_attr(unsigned long paddr, unsigned long size,
                err = set_memory_wb(vaddr, nrpages);
                break;
        }
-       if (!err)
-               global_flush_tlb();
 
        return err;
 }

diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index e4d2b69..a2d747c 100644 (file)
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -23,6 +23,36 @@ within(unsigned long addr, unsigned long start, unsigned long end)
 }
 
 /*
+ * Flushing functions
+ */
+void clflush_cache_range(void *addr, int size)
+{
+       int i;
+
+       for (i = 0; i < size; i += boot_cpu_data.x86_clflush_size)
+               clflush(addr+i);
+}
+
+static void flush_kernel_map(void *arg)
+{
+       /*
+        * Flush all to work around Errata in early athlons regarding
+        * large page flushing.
+        */
+       __flush_tlb_all();
+
+       if (boot_cpu_data.x86_model >= 4)
+               wbinvd();
+}
+
+static void global_flush_tlb(void)
+{
+       BUG_ON(irqs_disabled());
+
+       on_each_cpu(flush_kernel_map, NULL, 1, 1);
+}
+
+/*
  * Certain areas of memory on x86 require very specific protection flags,
  * for example the BIOS area or kernel text. Callers don't always get this
  * right (again, ioremap() on BIOS memory is not uncommon) so this function
@@ -328,149 +358,124 @@ static int change_page_attr_clear(unsigned long addr, int numpages,
 
 int set_memory_uc(unsigned long addr, int numpages)
 {
-       pgprot_t uncached;
+       int err;
 
-       pgprot_val(uncached) = _PAGE_PCD | _PAGE_PWT;
-       return change_page_attr_set(addr, numpages, uncached);
+       err = change_page_attr_set(addr, numpages,
+                               __pgprot(_PAGE_PCD | _PAGE_PWT));
+       global_flush_tlb();
+       return err;
 }
 EXPORT_SYMBOL(set_memory_uc);
 
 int set_memory_wb(unsigned long addr, int numpages)
 {
-       pgprot_t uncached;
+       int err;
 
-       pgprot_val(uncached) = _PAGE_PCD | _PAGE_PWT;
-       return change_page_attr_clear(addr, numpages, uncached);
+       err = change_page_attr_clear(addr, numpages,
+                               __pgprot(_PAGE_PCD | _PAGE_PWT));
+       global_flush_tlb();
+       return err;
 }
 EXPORT_SYMBOL(set_memory_wb);
 
 int set_memory_x(unsigned long addr, int numpages)
 {
-       pgprot_t nx;
+       int err;
 
-       pgprot_val(nx) = _PAGE_NX;
-       return change_page_attr_clear(addr, numpages, nx);
+       err = change_page_attr_clear(addr, numpages,
+                               __pgprot(_PAGE_NX));
+       global_flush_tlb();
+       return err;
 }
 EXPORT_SYMBOL(set_memory_x);
 
 int set_memory_nx(unsigned long addr, int numpages)
 {
-       pgprot_t nx;
+       int err;
 
-       pgprot_val(nx) = _PAGE_NX;
-       return change_page_attr_set(addr, numpages, nx);
+       err = change_page_attr_set(addr, numpages,
+                               __pgprot(_PAGE_NX));
+       global_flush_tlb();
+       return err;
 }
 EXPORT_SYMBOL(set_memory_nx);
 
 int set_memory_ro(unsigned long addr, int numpages)
 {
-       pgprot_t rw;
+       int err;
 
-       pgprot_val(rw) = _PAGE_RW;
-       return change_page_attr_clear(addr, numpages, rw);
+       err = change_page_attr_clear(addr, numpages,
+                               __pgprot(_PAGE_RW));
+       global_flush_tlb();
+       return err;
 }
 
 int set_memory_rw(unsigned long addr, int numpages)
 {
-       pgprot_t rw;
+       int err;
 
-       pgprot_val(rw) = _PAGE_RW;
-       return change_page_attr_set(addr, numpages, rw);
+       err = change_page_attr_set(addr, numpages,
+                               __pgprot(_PAGE_RW));
+       global_flush_tlb();
+       return err;
 }
 
 int set_memory_np(unsigned long addr, int numpages)
 {
-       pgprot_t present;
+       int err;
 
-       pgprot_val(present) = _PAGE_PRESENT;
-       return change_page_attr_clear(addr, numpages, present);
+       err = change_page_attr_clear(addr, numpages,
+                               __pgprot(_PAGE_PRESENT));
+       global_flush_tlb();
+       return err;
 }
 
 int set_pages_uc(struct page *page, int numpages)
 {
        unsigned long addr = (unsigned long)page_address(page);
-       pgprot_t uncached;
 
-       pgprot_val(uncached) = _PAGE_PCD | _PAGE_PWT;
-       return change_page_attr_set(addr, numpages, uncached);
+       return set_memory_uc(addr, numpages);
 }
 EXPORT_SYMBOL(set_pages_uc);
 
 int set_pages_wb(struct page *page, int numpages)
 {
        unsigned long addr = (unsigned long)page_address(page);
-       pgprot_t uncached;
 
-       pgprot_val(uncached) = _PAGE_PCD | _PAGE_PWT;
-       return change_page_attr_clear(addr, numpages, uncached);
+       return set_memory_wb(addr, numpages);
 }
 EXPORT_SYMBOL(set_pages_wb);
 
 int set_pages_x(struct page *page, int numpages)
 {
        unsigned long addr = (unsigned long)page_address(page);
-       pgprot_t nx;
 
-       pgprot_val(nx) = _PAGE_NX;
-       return change_page_attr_clear(addr, numpages, nx);
+       return set_memory_x(addr, numpages);
 }
 EXPORT_SYMBOL(set_pages_x);
 
 int set_pages_nx(struct page *page, int numpages)
 {
        unsigned long addr = (unsigned long)page_address(page);
-       pgprot_t nx;
 
-       pgprot_val(nx) = _PAGE_NX;
-       return change_page_attr_set(addr, numpages, nx);
+       return set_memory_nx(addr, numpages);
 }
 EXPORT_SYMBOL(set_pages_nx);
 
 int set_pages_ro(struct page *page, int numpages)
 {
        unsigned long addr = (unsigned long)page_address(page);
-       pgprot_t rw;
 
-       pgprot_val(rw) = _PAGE_RW;
-       return change_page_attr_clear(addr, numpages, rw);
+       return set_memory_ro(addr, numpages);
 }
 
 int set_pages_rw(struct page *page, int numpages)
 {
        unsigned long addr = (unsigned long)page_address(page);
-       pgprot_t rw;
-
-       pgprot_val(rw) = _PAGE_RW;
-       return change_page_attr_set(addr, numpages, rw);
-}
-
-void clflush_cache_range(void *addr, int size)
-{
-       int i;
-
-       for (i = 0; i < size; i += boot_cpu_data.x86_clflush_size)
-               clflush(addr+i);
-}
 
-static void flush_kernel_map(void *arg)
-{
-       /*
-        * Flush all to work around Errata in early athlons regarding
-        * large page flushing.
-        */
-       __flush_tlb_all();
-
-       if (boot_cpu_data.x86_model >= 4)
-               wbinvd();
+       return set_memory_rw(addr, numpages);
 }
 
-void global_flush_tlb(void)
-{
-       BUG_ON(irqs_disabled());
-
-       on_each_cpu(flush_kernel_map, NULL, 1, 1);
-}
-EXPORT_SYMBOL(global_flush_tlb);
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
 

diff --git a/drivers/char/agp/ali-agp.c b/drivers/char/agp/ali-agp.c
index aa5ddb7..1ffb381 100644 (file)
--- a/drivers/char/agp/ali-agp.c
+++ b/drivers/char/agp/ali-agp.c
@@ -145,7 +145,6 @@ static void *m1541_alloc_page(struct agp_bridge_data *bridge)
        void *addr = agp_generic_alloc_page(agp_bridge);
        u32 temp;
 
-       global_flush_tlb();
        if (!addr)
                return NULL;
 
@@ -162,7 +161,6 @@ static void ali_destroy_page(void * addr, int flags)
                if (flags & AGP_PAGE_DESTROY_UNMAP) {
                        global_cache_flush();   /* is this really needed?  --hch */
                        agp_generic_destroy_page(addr, flags);
-                       global_flush_tlb();
                } else
                        agp_generic_destroy_page(addr, flags);
        }

diff --git a/drivers/char/agp/i460-agp.c b/drivers/char/agp/i460-agp.c
index e72a83e..76f581c 100644 (file)
--- a/drivers/char/agp/i460-agp.c
+++ b/drivers/char/agp/i460-agp.c
@@ -527,7 +527,6 @@ static void *i460_alloc_page (struct agp_bridge_data *bridge)
 
        if (I460_IO_PAGE_SHIFT <= PAGE_SHIFT) {
                page = agp_generic_alloc_page(agp_bridge);
-               global_flush_tlb();
        } else
                /* Returning NULL would cause problems */
                /* AK: really dubious code. */
@@ -539,7 +538,6 @@ static void i460_destroy_page (void *page, int flags)
 {
        if (I460_IO_PAGE_SHIFT <= PAGE_SHIFT) {
                agp_generic_destroy_page(page, flags);
-               global_flush_tlb();
        }
 }
 

diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index c03a714..189efb6 100644 (file)
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -212,11 +212,9 @@ static void *i8xx_alloc_pages(void)
 
        if (set_pages_uc(page, 4) < 0) {
                set_pages_wb(page, 4);
-               global_flush_tlb();
                __free_pages(page, 2);
                return NULL;
        }
-       global_flush_tlb();
        get_page(page);
        atomic_inc(&agp_bridge->current_memory_agp);
        return page_address(page);
@@ -231,7 +229,6 @@ static void i8xx_destroy_pages(void *addr)
 
        page = virt_to_page(addr);
        set_pages_wb(page, 4);
-       global_flush_tlb();
        put_page(page);
        __free_pages(page, 2);
        atomic_dec(&agp_bridge->current_memory_agp);
@@ -341,7 +338,6 @@ static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 
        switch (pg_count) {
        case 1: addr = agp_bridge->driver->agp_alloc_page(agp_bridge);
-               global_flush_tlb();
                break;
        case 4:
                /* kludge to get 4 physical pages for ARGB cursor */
@@ -404,7 +400,6 @@ static void intel_i810_free_by_type(struct agp_memory *curr)
                else {
                        agp_bridge->driver->agp_destroy_page(gart_to_virt(curr->memory[0]),
                                                             AGP_PAGE_DESTROY_UNMAP);
-                       global_flush_tlb();
                        agp_bridge->driver->agp_destroy_page(gart_to_virt(curr->memory[0]),
                                                             AGP_PAGE_DESTROY_FREE);
                }

diff --git a/drivers/video/vermilion/vermilion.c b/drivers/video/vermilion/vermilion.c
index fb72778..1c65666 100644 (file)
--- a/drivers/video/vermilion/vermilion.c
+++ b/drivers/video/vermilion/vermilion.c
@@ -124,13 +124,8 @@ static int vmlfb_alloc_vram_area(struct vram_area *va, unsigned max_order,
        /*
         * Change caching policy of the linear kernel map to avoid
         * mapping type conflicts with user-space mappings.
-        * The first global_flush_tlb() is really only there to do a global
-        * wbinvd().
         */
-
-       global_flush_tlb();
        set_pages_uc(virt_to_page(va->logical), va->size >> PAGE_SHIFT);
-       global_flush_tlb();
 
        printk(KERN_DEBUG MODULE_NAME
               ": Allocated %ld bytes vram area at 0x%08lx\n",
@@ -156,7 +151,6 @@ static void vmlfb_free_vram_area(struct vram_area *va)
 
                set_pages_wb(virt_to_page(va->logical),
                                 va->size >> PAGE_SHIFT);
-               global_flush_tlb();
 
                /*
                 * Decrease the usage count on the pages we've used

diff --git a/include/asm-x86/agp.h b/include/asm-x86/agp.h
index f6df725..0c309b9 100644 (file)
--- a/include/asm-x86/agp.h
+++ b/include/asm-x86/agp.h
@@ -12,13 +12,9 @@
  * page. This avoids data corruption on some CPUs.
  */
 
-/*
- * Caller's responsibility to call global_flush_tlb() for performance
- * reasons
- */
 #define map_page_into_agp(page) set_pages_uc(page, 1)
 #define unmap_page_from_agp(page) set_pages_wb(page, 1)
-#define flush_agp_mappings() global_flush_tlb()
+#define flush_agp_mappings() do { } while (0)
 
 /*
  * Could use CLFLUSH here if the cpu supports it. But then it would

diff --git a/include/asm-x86/cacheflush.h b/include/asm-x86/cacheflush.h
index d15ff35..157da02 100644 (file)
--- a/include/asm-x86/cacheflush.h
+++ b/include/asm-x86/cacheflush.h
@@ -24,7 +24,6 @@
 #define copy_from_user_page(vma, page, vaddr, dst, src, len) \
        memcpy(dst, src, len)
 
-void global_flush_tlb(void);
 int __deprecated_for_modules change_page_attr(struct page *page, int numpages,
                                                                pgprot_t prot);
 

diff --git a/sound/pci/intel8x0.c b/sound/pci/intel8x0.c
index e565090..4bb9764 100644 (file)
--- a/sound/pci/intel8x0.c
+++ b/sound/pci/intel8x0.c
@@ -715,7 +715,6 @@ static void fill_nocache(void *buf, int size, int nocache)
                set_pages_uc(virt_to_page(buf), size);
        else
                set_pages_wb(virt_to_page(buf), size);
-       global_flush_tlb();
 }
 #else
 #define fill_nocache(buf, size, nocache) do { ; } while (0)