ARM: dma-mapping: push buffer ownership down into dma-mapping.c

Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Tested-By: Santosh Shilimkar <santosh.shilimkar@ti.com>

diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index e850f5c..256ee1c 100644 (file)
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -57,46 +57,57 @@ static inline dma_addr_t virt_to_dma(struct device *dev, void *addr)
 #endif
 
 /*
- * Private support functions: these are not part of the API and are
- * liable to change.  Drivers must not use these.
- */
-extern void dma_cache_maint(const void *kaddr, size_t size, int rw);
-extern void dma_cache_maint_page(struct page *page, unsigned long offset,
-                                size_t size, int rw);
-
-/*
  * The DMA API is built upon the notion of "buffer ownership".  A buffer
  * is either exclusively owned by the CPU (and therefore may be accessed
  * by it) or exclusively owned by the DMA device.  These helper functions
  * represent the transitions between these two ownership states.
  *
- * As above, these are private support functions and not part of the API.
- * Drivers must not use these.
+ * Note, however, that on later ARMs, this notion does not work due to
+ * speculative prefetches.  We model our approach on the assumption that
+ * the CPU does do speculative prefetches, which means we clean caches
+ * before transfers and delay cache invalidation until transfer completion.
+ *
+ * Private support functions: these are not part of the API and are
+ * liable to change.  Drivers must not use these.
  */
 static inline void __dma_single_cpu_to_dev(const void *kaddr, size_t size,
        enum dma_data_direction dir)
 {
+       extern void ___dma_single_cpu_to_dev(const void *, size_t,
+               enum dma_data_direction);
+
        if (!arch_is_coherent())
-               dma_cache_maint(kaddr, size, dir);
+               ___dma_single_cpu_to_dev(kaddr, size, dir);
 }
 
 static inline void __dma_single_dev_to_cpu(const void *kaddr, size_t size,
        enum dma_data_direction dir)
 {
-       /* nothing to do */
+       extern void ___dma_single_dev_to_cpu(const void *, size_t,
+               enum dma_data_direction);
+
+       if (!arch_is_coherent())
+               ___dma_single_dev_to_cpu(kaddr, size, dir);
 }
 
 static inline void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
        size_t size, enum dma_data_direction dir)
 {
+       extern void ___dma_page_cpu_to_dev(struct page *, unsigned long,
+               size_t, enum dma_data_direction);
+
        if (!arch_is_coherent())
-               dma_cache_maint_page(page, off, size, dir);
+               ___dma_page_cpu_to_dev(page, off, size, dir);
 }
 
 static inline void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
        size_t size, enum dma_data_direction dir)
 {
-       /* nothing to do */
+       extern void ___dma_page_dev_to_cpu(struct page *, unsigned long,
+               size_t, enum dma_data_direction);
+
+       if (!arch_is_coherent())
+               ___dma_page_dev_to_cpu(page, off, size, dir);
 }
 
 /*

diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index a316c94..bbf8788 100644 (file)
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -404,7 +404,7 @@ EXPORT_SYMBOL(dma_free_coherent);
  * platforms with CONFIG_DMABOUNCE.
  * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
  */
-void dma_cache_maint(const void *start, size_t size, int direction)
+static void dma_cache_maint(const void *start, size_t size, int direction)
 {
        void (*inner_op)(const void *, const void *);
        void (*outer_op)(unsigned long, unsigned long);
@@ -431,7 +431,20 @@ void dma_cache_maint(const void *start, size_t size, int direction)
        inner_op(start, start + size);
        outer_op(__pa(start), __pa(start) + size);
 }
-EXPORT_SYMBOL(dma_cache_maint);
+
+void ___dma_single_cpu_to_dev(const void *kaddr, size_t size,
+       enum dma_data_direction dir)
+{
+       dma_cache_maint(kaddr, size, dir);
+}
+EXPORT_SYMBOL(___dma_single_cpu_to_dev);
+
+void ___dma_single_dev_to_cpu(const void *kaddr, size_t size,
+       enum dma_data_direction dir)
+{
+       /* nothing to do */
+}
+EXPORT_SYMBOL(___dma_single_dev_to_cpu);
 
 static void dma_cache_maint_contiguous(struct page *page, unsigned long offset,
                                       size_t size, int direction)
@@ -474,7 +487,7 @@ static void dma_cache_maint_contiguous(struct page *page, unsigned long offset,
        outer_op(paddr, paddr + size);
 }
 
-void dma_cache_maint_page(struct page *page, unsigned long offset,
+static void dma_cache_maint_page(struct page *page, unsigned long offset,
                          size_t size, int dir)
 {
        /*
@@ -499,7 +512,20 @@ void dma_cache_maint_page(struct page *page, unsigned long offset,
                left -= len;
        } while (left);
 }
-EXPORT_SYMBOL(dma_cache_maint_page);
+
+void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
+       size_t size, enum dma_data_direction dir)
+{
+       dma_cache_maint_page(page, off, size, dir);
+}
+EXPORT_SYMBOL(___dma_page_cpu_to_dev);
+
+void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
+       size_t size, enum dma_data_direction dir)
+{
+       /* nothing to do */
+}
+EXPORT_SYMBOL(___dma_page_dev_to_cpu);
 
 /**
  * dma_map_sg - map a set of SG buffers for streaming mode DMA