#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
-#define DMAR_OPERATION_TIMEOUT (HZ*60) /* 1m */
+#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) /* 10sec */
#define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
static struct intel_iommu *g_iommus;
+
+#define HIGH_WATER_MARK 250
+struct deferred_flush_tables {
+ int next;
+ struct iova *iova[HIGH_WATER_MARK];
+ struct dmar_domain *domain[HIGH_WATER_MARK];
+};
+
+static struct deferred_flush_tables *deferred_flush;
+
/* bitmap for indexing intel_iommus */
-static unsigned long *g_iommus_to_flush;
static int g_num_of_iommus;
static DEFINE_SPINLOCK(async_umap_flush_lock);
static int timer_on;
static long list_size;
-static int high_watermark;
-
-static struct dentry *intel_iommu_debug, *debug;
-
static void domain_remove_dev_info(struct dmar_domain *domain);
#define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \
{\
- unsigned long start_time = jiffies;\
+ cycles_t start_time = get_cycles();\
while (1) {\
sts = op (iommu->reg + offset);\
if (cond)\
break;\
- if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))\
+ if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start_time))\
panic("DMAR hardware is malfunctioning\n");\
cpu_relax();\
}\
}
#ifdef CONFIG_DMAR_GFX_WA
-extern int arch_get_ram_range(int slot, u64 *addr, u64 *size);
+struct iommu_prepare_data {
+ struct pci_dev *pdev;
+ int ret;
+};
+
+static int __init iommu_prepare_work_fn(unsigned long start_pfn,
+ unsigned long end_pfn, void *datax)
+{
+ struct iommu_prepare_data *data;
+
+ data = (struct iommu_prepare_data *)datax;
+
+ data->ret = iommu_prepare_identity_map(data->pdev,
+ start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
+ return data->ret;
+
+}
+
+static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
+{
+ int nid;
+ struct iommu_prepare_data data;
+
+ data.pdev = pdev;
+ data.ret = 0;
+
+ for_each_online_node(nid) {
+ work_with_active_regions(nid, iommu_prepare_work_fn, &data);
+ if (data.ret)
+ return data.ret;
+ }
+ return data.ret;
+}
+
static void __init iommu_prepare_gfx_mapping(void)
{
struct pci_dev *pdev = NULL;
- u64 base, size;
- int slot;
int ret;
for_each_pci_dev(pdev) {
continue;
printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
pci_name(pdev));
- slot = arch_get_ram_range(0, &base, &size);
- while (slot >= 0) {
- ret = iommu_prepare_identity_map(pdev,
- base, base + size);
- if (ret)
- goto error;
- slot = arch_get_ram_range(slot, &base, &size);
- }
- continue;
-error:
- printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
+ ret = iommu_prepare_with_active_regions(pdev);
+ if (ret)
+ printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
}
}
#endif
struct dmar_rmrr_unit *rmrr;
struct pci_dev *pdev;
struct intel_iommu *iommu;
- int nlongs, i, ret, unit = 0;
+ int i, ret, unit = 0;
/*
* for each drhd
*/
}
- nlongs = BITS_TO_LONGS(g_num_of_iommus);
- g_iommus_to_flush = kzalloc(nlongs * sizeof(unsigned long), GFP_KERNEL);
- if (!g_iommus_to_flush) {
- printk(KERN_ERR "Intel-IOMMU: "
- "Allocating bitmap array failed\n");
- return -ENOMEM;
- }
-
g_iommus = kzalloc(g_num_of_iommus * sizeof(*iommu), GFP_KERNEL);
if (!g_iommus) {
- kfree(g_iommus_to_flush);
+ ret = -ENOMEM;
+ goto error;
+ }
+
+ deferred_flush = kzalloc(g_num_of_iommus *
+ sizeof(struct deferred_flush_tables), GFP_KERNEL);
+ if (!deferred_flush) {
ret = -ENOMEM;
goto error;
}
return domain;
}
-static dma_addr_t intel_map_single(struct device *hwdev, void *addr,
- size_t size, int dir)
+static dma_addr_t
+intel_map_single(struct device *hwdev, phys_addr_t paddr, size_t size, int dir)
{
struct pci_dev *pdev = to_pci_dev(hwdev);
- int ret;
struct dmar_domain *domain;
- unsigned long start_addr;
+ unsigned long start_paddr;
struct iova *iova;
int prot = 0;
+ int ret;
BUG_ON(dir == DMA_NONE);
if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
- return virt_to_bus(addr);
+ return paddr;
domain = get_valid_domain_for_dev(pdev);
if (!domain)
return 0;
- addr = (void *)virt_to_phys(addr);
- size = aligned_size((u64)addr, size);
+ size = aligned_size((u64)paddr, size);
iova = __intel_alloc_iova(hwdev, domain, size);
if (!iova)
goto error;
- start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
+ start_paddr = iova->pfn_lo << PAGE_SHIFT_4K;
/*
* Check if DMAR supports zero-length reads on write only
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
prot |= DMA_PTE_WRITE;
/*
- * addr - (addr + size) might be partial page, we should map the whole
+ * paddr - (paddr + size) might be partial page, we should map the whole
* page. Note: if two part of one page are separately mapped, we
- * might have two guest_addr mapping to the same host addr, but this
+ * might have two guest_addr mapping to the same host paddr, but this
* is not a big problem
*/
- ret = domain_page_mapping(domain, start_addr,
- ((u64)addr) & PAGE_MASK_4K, size, prot);
+ ret = domain_page_mapping(domain, start_paddr,
+ ((u64)paddr) & PAGE_MASK_4K, size, prot);
if (ret)
goto error;
pr_debug("Device %s request: %lx@%llx mapping: %lx@%llx, dir %d\n",
- pci_name(pdev), size, (u64)addr,
- size, (u64)start_addr, dir);
+ pci_name(pdev), size, (u64)paddr,
+ size, (u64)start_paddr, dir);
/* it's a non-present to present mapping */
ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
- start_addr, size >> PAGE_SHIFT_4K, 1);
+ start_paddr, size >> PAGE_SHIFT_4K, 1);
if (ret)
iommu_flush_write_buffer(domain->iommu);
- return (start_addr + ((u64)addr & (~PAGE_MASK_4K)));
+ return (start_paddr + ((u64)paddr & (~PAGE_MASK_4K)));
error:
if (iova)
__free_iova(&domain->iovad, iova);
printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
- pci_name(pdev), size, (u64)addr, dir);
+ pci_name(pdev), size, (u64)paddr, dir);
return 0;
}
static void flush_unmaps(void)
{
- struct iova *node, *n;
- unsigned long flags;
- int i;
+ int i, j;
- spin_lock_irqsave(&async_umap_flush_lock, flags);
timer_on = 0;
/* just flush them all */
for (i = 0; i < g_num_of_iommus; i++) {
- if (test_and_clear_bit(i, g_iommus_to_flush))
+ if (deferred_flush[i].next) {
iommu_flush_iotlb_global(&g_iommus[i], 0);
+ for (j = 0; j < deferred_flush[i].next; j++) {
+ __free_iova(&deferred_flush[i].domain[j]->iovad,
+ deferred_flush[i].iova[j]);
+ }
+ deferred_flush[i].next = 0;
+ }
}
- list_for_each_entry_safe(node, n, &unmaps_to_do, list) {
- /* free iova */
- list_del(&node->list);
- __free_iova(&((struct dmar_domain *)node->dmar)->iovad, node);
-
- }
list_size = 0;
- spin_unlock_irqrestore(&async_umap_flush_lock, flags);
}
static void flush_unmaps_timeout(unsigned long data)
{
+ unsigned long flags;
+
+ spin_lock_irqsave(&async_umap_flush_lock, flags);
flush_unmaps();
+ spin_unlock_irqrestore(&async_umap_flush_lock, flags);
}
static void add_unmap(struct dmar_domain *dom, struct iova *iova)
{
unsigned long flags;
+ int next, iommu_id;
spin_lock_irqsave(&async_umap_flush_lock, flags);
- iova->dmar = dom;
- list_add(&iova->list, &unmaps_to_do);
- set_bit((dom->iommu - g_iommus), g_iommus_to_flush);
+ if (list_size == HIGH_WATER_MARK)
+ flush_unmaps();
+
+ iommu_id = dom->iommu - g_iommus;
+ next = deferred_flush[iommu_id].next;
+ deferred_flush[iommu_id].domain[next] = dom;
+ deferred_flush[iommu_id].iova[next] = iova;
+ deferred_flush[iommu_id].next++;
if (!timer_on) {
mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
* queue up the release of the unmap to save the 1/6th of the
* cpu used up by the iotlb flush operation...
*/
- if (list_size > high_watermark)
- flush_unmaps();
}
}
return NULL;
memset(vaddr, 0, size);
- *dma_handle = intel_map_single(hwdev, vaddr, size, DMA_BIDIRECTIONAL);
+ *dma_handle = intel_map_single(hwdev, virt_to_bus(vaddr), size, DMA_BIDIRECTIONAL);
if (*dma_handle)
return vaddr;
free_pages((unsigned long)vaddr, order);
if (dmar_table_init())
return -ENODEV;
- high_watermark = 250;
- intel_iommu_debug = debugfs_create_dir("intel_iommu", NULL);
- debug = debugfs_create_u32("high_watermark", S_IWUGO | S_IRUGO,
- intel_iommu_debug, &high_watermark);
iommu_init_mempool();
dmar_init_reserved_ranges();
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff