#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/dma-mapping.h>
-#include <linux/init.h>
#include <linux/bitops.h>
+#include <linux/iommu-helper.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/iommu.h>
static int novmerge = 1;
#endif
+static int protect4gb = 1;
+
static inline unsigned long iommu_num_pages(unsigned long vaddr,
unsigned long slen)
{
return npages;
}
+static int __init setup_protect4gb(char *str)
+{
+ if (strcmp(str, "on") == 0)
+ protect4gb = 1;
+ else if (strcmp(str, "off") == 0)
+ protect4gb = 0;
+
+ return 1;
+}
+
static int __init setup_iommu(char *str)
{
if (!strcmp(str, "novmerge"))
return 1;
}
+__setup("protect4gb=", setup_protect4gb);
__setup("iommu=", setup_iommu);
-static unsigned long iommu_range_alloc(struct iommu_table *tbl,
+static unsigned long iommu_range_alloc(struct device *dev,
+ struct iommu_table *tbl,
unsigned long npages,
unsigned long *handle,
unsigned long mask,
unsigned int align_order)
{
- unsigned long n, end, i, start;
+ unsigned long n, end, start;
unsigned long limit;
int largealloc = npages > 15;
int pass = 0;
unsigned long align_mask;
+ unsigned long boundary_size;
align_mask = 0xffffffffffffffffl >> (64 - align_order);
start &= mask;
}
- n = find_next_zero_bit(tbl->it_map, limit, start);
-
- /* Align allocation */
- n = (n + align_mask) & ~align_mask;
-
- end = n + npages;
+ if (dev)
+ boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
+ 1 << IOMMU_PAGE_SHIFT);
+ else
+ boundary_size = ALIGN(1UL << 32, 1 << IOMMU_PAGE_SHIFT);
+ /* 4GB boundary for iseries_hv_alloc and iseries_hv_map */
- if (unlikely(end >= limit)) {
+ n = iommu_area_alloc(tbl->it_map, limit, start, npages,
+ tbl->it_offset, boundary_size >> IOMMU_PAGE_SHIFT,
+ align_mask);
+ if (n == -1) {
if (likely(pass < 2)) {
/* First failure, just rescan the half of the table.
* Second failure, rescan the other half of the table.
}
}
- for (i = n; i < end; i++)
- if (test_bit(i, tbl->it_map)) {
- start = i+1;
- goto again;
- }
-
- for (i = n; i < end; i++)
- __set_bit(i, tbl->it_map);
+ end = n + npages;
/* Bump the hint to a new block for small allocs. */
if (largealloc) {
return n;
}
-static dma_addr_t iommu_alloc(struct iommu_table *tbl, void *page,
- unsigned int npages, enum dma_data_direction direction,
- unsigned long mask, unsigned int align_order)
+static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
+ void *page, unsigned int npages,
+ enum dma_data_direction direction,
+ unsigned long mask, unsigned int align_order)
{
unsigned long entry, flags;
dma_addr_t ret = DMA_ERROR_CODE;
spin_lock_irqsave(&(tbl->it_lock), flags);
- entry = iommu_range_alloc(tbl, npages, NULL, mask, align_order);
+ entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order);
if (unlikely(entry == DMA_ERROR_CODE)) {
spin_unlock_irqrestore(&(tbl->it_lock), flags);
unsigned int npages)
{
unsigned long entry, free_entry;
- unsigned long i;
entry = dma_addr >> IOMMU_PAGE_SHIFT;
free_entry = entry - tbl->it_offset;
}
ppc_md.tce_free(tbl, entry, npages);
-
- for (i = 0; i < npages; i++)
- __clear_bit(free_entry+i, tbl->it_map);
+ iommu_area_free(tbl->it_map, free_entry, npages);
}
static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
spin_unlock_irqrestore(&(tbl->it_lock), flags);
}
-int iommu_map_sg(struct iommu_table *tbl, struct scatterlist *sglist,
- int nelems, unsigned long mask,
- enum dma_data_direction direction)
+int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
+ struct scatterlist *sglist, int nelems,
+ unsigned long mask, enum dma_data_direction direction,
+ struct dma_attrs *attrs)
{
dma_addr_t dma_next = 0, dma_addr;
unsigned long flags;
struct scatterlist *s, *outs, *segstart;
- int outcount, incount;
+ int outcount, incount, i;
+ unsigned int align;
unsigned long handle;
+ unsigned int max_seg_size;
BUG_ON(direction == DMA_NONE);
spin_lock_irqsave(&(tbl->it_lock), flags);
- for (s = outs; nelems; nelems--, s++) {
+ max_seg_size = dma_get_max_seg_size(dev);
+ for_each_sg(sglist, s, nelems, i) {
unsigned long vaddr, npages, entry, slen;
slen = s->length;
continue;
}
/* Allocate iommu entries for that segment */
- vaddr = (unsigned long)page_address(s->page) + s->offset;
+ vaddr = (unsigned long) sg_virt(s);
npages = iommu_num_pages(vaddr, slen);
- entry = iommu_range_alloc(tbl, npages, &handle, mask >> IOMMU_PAGE_SHIFT, 0);
+ align = 0;
+ if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && slen >= PAGE_SIZE &&
+ (vaddr & ~PAGE_MASK) == 0)
+ align = PAGE_SHIFT - IOMMU_PAGE_SHIFT;
+ entry = iommu_range_alloc(dev, tbl, npages, &handle,
+ mask >> IOMMU_PAGE_SHIFT, align);
DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen);
/* We cannot merge if:
* - allocated dma_addr isn't contiguous to previous allocation
*/
- if (novmerge || (dma_addr != dma_next)) {
+ if (novmerge || (dma_addr != dma_next) ||
+ (outs->dma_length + s->length > max_seg_size)) {
/* Can't merge: create a new segment */
segstart = s;
- outcount++; outs++;
+ outcount++;
+ outs = sg_next(outs);
DBG(" can't merge, new segment.\n");
} else {
outs->dma_length += s->length;
* next entry of the sglist if we didn't fill the list completely
*/
if (outcount < incount) {
- outs++;
+ outs = sg_next(outs);
outs->dma_address = DMA_ERROR_CODE;
outs->dma_length = 0;
}
return outcount;
failure:
- for (s = &sglist[0]; s <= outs; s++) {
+ for_each_sg(sglist, s, nelems, i) {
if (s->dma_length != 0) {
unsigned long vaddr, npages;
s->dma_address = DMA_ERROR_CODE;
s->dma_length = 0;
}
+ if (s == outs)
+ break;
}
spin_unlock_irqrestore(&(tbl->it_lock), flags);
return 0;
void iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist,
- int nelems, enum dma_data_direction direction)
+ int nelems, enum dma_data_direction direction,
+ struct dma_attrs *attrs)
{
+ struct scatterlist *sg;
unsigned long flags;
BUG_ON(direction == DMA_NONE);
spin_lock_irqsave(&(tbl->it_lock), flags);
+ sg = sglist;
while (nelems--) {
unsigned int npages;
- dma_addr_t dma_handle = sglist->dma_address;
+ dma_addr_t dma_handle = sg->dma_address;
- if (sglist->dma_length == 0)
+ if (sg->dma_length == 0)
break;
- npages = iommu_num_pages(dma_handle,sglist->dma_length);
+ npages = iommu_num_pages(dma_handle, sg->dma_length);
__iommu_free(tbl, dma_handle, npages);
- sglist++;
+ sg = sg_next(sg);
}
/* Flush/invalidate TLBs if necessary. As for iommu_free(), we
#ifdef CONFIG_CRASH_DUMP
if (ppc_md.tce_get) {
- unsigned long index, tceval;
+ unsigned long index;
+ unsigned long tceval;
unsigned long tcecount = 0;
/*
return tbl;
}
-void iommu_free_table(struct device_node *dn)
+void iommu_free_table(struct iommu_table *tbl, const char *node_name)
{
- struct pci_dn *pdn = dn->data;
- struct iommu_table *tbl = pdn->iommu_table;
unsigned long bitmap_sz, i;
unsigned int order;
if (!tbl || !tbl->it_map) {
- printk(KERN_ERR "%s: expected TCE map for %s\n", __FUNCTION__,
- dn->full_name);
+ printk(KERN_ERR "%s: expected TCE map for %s\n", __func__,
+ node_name);
return;
}
for (i = 0; i < (tbl->it_size/64); i++) {
if (tbl->it_map[i] != 0) {
printk(KERN_WARNING "%s: Unexpected TCEs for %s\n",
- __FUNCTION__, dn->full_name);
+ __func__, node_name);
break;
}
}
* need not be page aligned, the dma_addr_t returned will point to the same
* byte within the page as vaddr.
*/
-dma_addr_t iommu_map_single(struct iommu_table *tbl, void *vaddr,
- size_t size, unsigned long mask,
- enum dma_data_direction direction)
+dma_addr_t iommu_map_single(struct device *dev, struct iommu_table *tbl,
+ void *vaddr, size_t size, unsigned long mask,
+ enum dma_data_direction direction, struct dma_attrs *attrs)
{
dma_addr_t dma_handle = DMA_ERROR_CODE;
unsigned long uaddr;
- unsigned int npages;
+ unsigned int npages, align;
BUG_ON(direction == DMA_NONE);
npages = iommu_num_pages(uaddr, size);
if (tbl) {
- dma_handle = iommu_alloc(tbl, vaddr, npages, direction,
- mask >> IOMMU_PAGE_SHIFT, 0);
+ align = 0;
+ if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && size >= PAGE_SIZE &&
+ ((unsigned long)vaddr & ~PAGE_MASK) == 0)
+ align = PAGE_SHIFT - IOMMU_PAGE_SHIFT;
+
+ dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction,
+ mask >> IOMMU_PAGE_SHIFT, align);
if (dma_handle == DMA_ERROR_CODE) {
if (printk_ratelimit()) {
printk(KERN_INFO "iommu_alloc failed, "
}
void iommu_unmap_single(struct iommu_table *tbl, dma_addr_t dma_handle,
- size_t size, enum dma_data_direction direction)
+ size_t size, enum dma_data_direction direction,
+ struct dma_attrs *attrs)
{
unsigned int npages;
* Returns the virtual address of the buffer and sets dma_handle
* to the dma address (mapping) of the first page.
*/
-void *iommu_alloc_coherent(struct iommu_table *tbl, size_t size,
- dma_addr_t *dma_handle, unsigned long mask, gfp_t flag, int node)
+void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl,
+ size_t size, dma_addr_t *dma_handle,
+ unsigned long mask, gfp_t flag, int node)
{
void *ret = NULL;
dma_addr_t mapping;
/* Set up tces to cover the allocated range */
nio_pages = size >> IOMMU_PAGE_SHIFT;
io_order = get_iommu_order(size);
- mapping = iommu_alloc(tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
+ mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
mask >> IOMMU_PAGE_SHIFT, io_order);
if (mapping == DMA_ERROR_CODE) {
free_pages((unsigned long)ret, order);
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff