IB: Add DMA mapping functions to allow device drivers to interpose

The QLogic InfiniPath HCAs use programmed I/O instead of HW DMA.
This patch allows a verbs device driver to interpose on DMA mapping
function calls in order to avoid relying on bus_to_virt() and
phys_to_virt() to undo the mappings created by dma_map_single(),
dma_map_sg(), etc.

Signed-off-by: Ralph Campbell <ralph.campbell@qlogic.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 8eacc35..fd2353f 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -43,6 +43,8 @@
 
 #include <linux/types.h>
 #include <linux/device.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
 
 #include <asm/atomic.h>
 #include <asm/scatterlist.h>
@@ -848,6 +850,49 @@ struct ib_cache {
        u8                     *lmc_cache;
 };
 
+struct ib_dma_mapping_ops {
+       int             (*mapping_error)(struct ib_device *dev,
+                                        u64 dma_addr);
+       u64             (*map_single)(struct ib_device *dev,
+                                     void *ptr, size_t size,
+                                     enum dma_data_direction direction);
+       void            (*unmap_single)(struct ib_device *dev,
+                                       u64 addr, size_t size,
+                                       enum dma_data_direction direction);
+       u64             (*map_page)(struct ib_device *dev,
+                                   struct page *page, unsigned long offset,
+                                   size_t size,
+                                   enum dma_data_direction direction);
+       void            (*unmap_page)(struct ib_device *dev,
+                                     u64 addr, size_t size,
+                                     enum dma_data_direction direction);
+       int             (*map_sg)(struct ib_device *dev,
+                                 struct scatterlist *sg, int nents,
+                                 enum dma_data_direction direction);
+       void            (*unmap_sg)(struct ib_device *dev,
+                                   struct scatterlist *sg, int nents,
+                                   enum dma_data_direction direction);
+       u64             (*dma_address)(struct ib_device *dev,
+                                      struct scatterlist *sg);
+       unsigned int    (*dma_len)(struct ib_device *dev,
+                                  struct scatterlist *sg);
+       void            (*sync_single_for_cpu)(struct ib_device *dev,
+                                              u64 dma_handle,
+                                              size_t size,
+                                              enum dma_data_direction dir);
+       void            (*sync_single_for_device)(struct ib_device *dev,
+                                                 u64 dma_handle,
+                                                 size_t size,
+                                                 enum dma_data_direction dir);
+       void            *(*alloc_coherent)(struct ib_device *dev,
+                                          size_t size,
+                                          u64 *dma_handle,
+                                          gfp_t flag);
+       void            (*free_coherent)(struct ib_device *dev,
+                                        size_t size, void *cpu_addr,
+                                        u64 dma_handle);
+};
+
 struct iw_cm_verbs;
 
 struct ib_device {
@@ -992,6 +1037,8 @@ struct ib_device {
                                                  struct ib_mad *in_mad,
                                                  struct ib_mad *out_mad);
 
+       struct ib_dma_mapping_ops   *dma_ops;
+
        struct module               *owner;
        struct class_device          class_dev;
        struct kobject               ports_parent;
@@ -1395,10 +1442,216 @@ static inline int ib_req_ncomp_notif(struct ib_cq *cq, int wc_cnt)
  *   usable for DMA.
  * @pd: The protection domain associated with the memory region.
  * @mr_access_flags: Specifies the memory access rights.
+ *
+ * Note that the ib_dma_*() functions defined below must be used
+ * to create/destroy addresses used with the Lkey or Rkey returned
+ * by ib_get_dma_mr().
  */
 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
 
 /**
+ * ib_dma_mapping_error - check a DMA addr for error
+ * @dev: The device for which the dma_addr was created
+ * @dma_addr: The DMA address to check
+ */
+static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
+{
+       return dev->dma_ops ?
+               dev->dma_ops->mapping_error(dev, dma_addr) :
+               dma_mapping_error(dma_addr);
+}
+
+/**
+ * ib_dma_map_single - Map a kernel virtual address to DMA address
+ * @dev: The device for which the dma_addr is to be created
+ * @cpu_addr: The kernel virtual address
+ * @size: The size of the region in bytes
+ * @direction: The direction of the DMA
+ */
+static inline u64 ib_dma_map_single(struct ib_device *dev,
+                                   void *cpu_addr, size_t size,
+                                   enum dma_data_direction direction)
+{
+       return dev->dma_ops ?
+               dev->dma_ops->map_single(dev, cpu_addr, size, direction) :
+               dma_map_single(dev->dma_device, cpu_addr, size, direction);
+}
+
+/**
+ * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single()
+ * @dev: The device for which the DMA address was created
+ * @addr: The DMA address
+ * @size: The size of the region in bytes
+ * @direction: The direction of the DMA
+ */
+static inline void ib_dma_unmap_single(struct ib_device *dev,
+                                      u64 addr, size_t size,
+                                      enum dma_data_direction direction)
+{
+       dev->dma_ops ?
+               dev->dma_ops->unmap_single(dev, addr, size, direction) :
+               dma_unmap_single(dev->dma_device, addr, size, direction);
+}
+
+/**
+ * ib_dma_map_page - Map a physical page to DMA address
+ * @dev: The device for which the dma_addr is to be created
+ * @page: The page to be mapped
+ * @offset: The offset within the page
+ * @size: The size of the region in bytes
+ * @direction: The direction of the DMA
+ */
+static inline u64 ib_dma_map_page(struct ib_device *dev,
+                                 struct page *page,
+                                 unsigned long offset,
+                                 size_t size,
+                                        enum dma_data_direction direction)
+{
+       return dev->dma_ops ?
+               dev->dma_ops->map_page(dev, page, offset, size, direction) :
+               dma_map_page(dev->dma_device, page, offset, size, direction);
+}
+
+/**
+ * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page()
+ * @dev: The device for which the DMA address was created
+ * @addr: The DMA address
+ * @size: The size of the region in bytes
+ * @direction: The direction of the DMA
+ */
+static inline void ib_dma_unmap_page(struct ib_device *dev,
+                                    u64 addr, size_t size,
+                                    enum dma_data_direction direction)
+{
+       dev->dma_ops ?
+               dev->dma_ops->unmap_page(dev, addr, size, direction) :
+               dma_unmap_page(dev->dma_device, addr, size, direction);
+}
+
+/**
+ * ib_dma_map_sg - Map a scatter/gather list to DMA addresses
+ * @dev: The device for which the DMA addresses are to be created
+ * @sg: The array of scatter/gather entries
+ * @nents: The number of scatter/gather entries
+ * @direction: The direction of the DMA
+ */
+static inline int ib_dma_map_sg(struct ib_device *dev,
+                               struct scatterlist *sg, int nents,
+                               enum dma_data_direction direction)
+{
+       return dev->dma_ops ?
+               dev->dma_ops->map_sg(dev, sg, nents, direction) :
+               dma_map_sg(dev->dma_device, sg, nents, direction);
+}
+
+/**
+ * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses
+ * @dev: The device for which the DMA addresses were created
+ * @sg: The array of scatter/gather entries
+ * @nents: The number of scatter/gather entries
+ * @direction: The direction of the DMA
+ */
+static inline void ib_dma_unmap_sg(struct ib_device *dev,
+                                  struct scatterlist *sg, int nents,
+                                  enum dma_data_direction direction)
+{
+       dev->dma_ops ?
+               dev->dma_ops->unmap_sg(dev, sg, nents, direction) :
+               dma_unmap_sg(dev->dma_device, sg, nents, direction);
+}
+
+/**
+ * ib_sg_dma_address - Return the DMA address from a scatter/gather entry
+ * @dev: The device for which the DMA addresses were created
+ * @sg: The scatter/gather entry
+ */
+static inline u64 ib_sg_dma_address(struct ib_device *dev,
+                                   struct scatterlist *sg)
+{
+       return dev->dma_ops ?
+               dev->dma_ops->dma_address(dev, sg) : sg_dma_address(sg);
+}
+
+/**
+ * ib_sg_dma_len - Return the DMA length from a scatter/gather entry
+ * @dev: The device for which the DMA addresses were created
+ * @sg: The scatter/gather entry
+ */
+static inline unsigned int ib_sg_dma_len(struct ib_device *dev,
+                                        struct scatterlist *sg)
+{
+       return dev->dma_ops ?
+               dev->dma_ops->dma_len(dev, sg) : sg_dma_len(sg);
+}
+
+/**
+ * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU
+ * @dev: The device for which the DMA address was created
+ * @addr: The DMA address
+ * @size: The size of the region in bytes
+ * @dir: The direction of the DMA
+ */
+static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev,
+                                             u64 addr,
+                                             size_t size,
+                                             enum dma_data_direction dir)
+{
+       dev->dma_ops ?
+               dev->dma_ops->sync_single_for_cpu(dev, addr, size, dir) :
+               dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
+}
+
+/**
+ * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device
+ * @dev: The device for which the DMA address was created
+ * @addr: The DMA address
+ * @size: The size of the region in bytes
+ * @dir: The direction of the DMA
+ */
+static inline void ib_dma_sync_single_for_device(struct ib_device *dev,
+                                                u64 addr,
+                                                size_t size,
+                                                enum dma_data_direction dir)
+{
+       dev->dma_ops ?
+               dev->dma_ops->sync_single_for_device(dev, addr, size, dir) :
+               dma_sync_single_for_device(dev->dma_device, addr, size, dir);
+}
+
+/**
+ * ib_dma_alloc_coherent - Allocate memory and map it for DMA
+ * @dev: The device for which the DMA address is requested
+ * @size: The size of the region to allocate in bytes
+ * @dma_handle: A pointer for returning the DMA address of the region
+ * @flag: memory allocator flags
+ */
+static inline void *ib_dma_alloc_coherent(struct ib_device *dev,
+                                          size_t size,
+                                          u64 *dma_handle,
+                                          gfp_t flag)
+{
+       return dev->dma_ops ?
+               dev->dma_ops->alloc_coherent(dev, size, dma_handle, flag) :
+               dma_alloc_coherent(dev->dma_device, size, dma_handle, flag);
+}
+
+/**
+ * ib_dma_free_coherent - Free memory allocated by ib_dma_alloc_coherent()
+ * @dev: The device for which the DMA addresses were allocated
+ * @size: The size of the region
+ * @cpu_addr: the address returned by ib_dma_alloc_coherent()
+ * @dma_handle: the DMA address returned by ib_dma_alloc_coherent()
+ */
+static inline void ib_dma_free_coherent(struct ib_device *dev,
+                                       size_t size, void *cpu_addr,
+                                       u64 dma_handle)
+{
+       dev->dma_ops ?
+               dev->dma_ops->free_coherent(dev, size, cpu_addr, dma_handle) :
+               dma_free_coherent(dev->dma_device, size, cpu_addr, dma_handle);
+}
+
+/**
  * ib_reg_phys_mr - Prepares a virtually addressed memory region for use
  *   by an HCA.
  * @pd: The protection domain associated assigned to the registered region.