* this program; if not, write to the Free Software Foundation, Inc., 59 Temple
* Place - Suite 330, Boston, MA 02111-1307 USA.
*
- * Copyright (C) Ashok Raj <ashok.raj@intel.com>
- * Copyright (C) Shaohua Li <shaohua.li@intel.com>
- * Copyright (C) Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
+ * Copyright (C) 2006-2008 Intel Corporation
+ * Author: Ashok Raj <ashok.raj@intel.com>
+ * Author: Shaohua Li <shaohua.li@intel.com>
+ * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
*/
#include <linux/init.h>
#include <linux/bitmap.h>
+#include <linux/debugfs.h>
#include <linux/slab.h>
#include <linux/irq.h>
#include <linux/interrupt.h>
#include <linux/dmar.h>
#include <linux/dma-mapping.h>
#include <linux/mempool.h>
+#include <linux/timer.h>
#include "iova.h"
#include "intel-iommu.h"
#include <asm/proto.h> /* force_iommu in this header in x86-64*/
#include <asm/cacheflush.h>
-#include <asm/iommu.h>
+#include <asm/gart.h>
#include "pci.h"
#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
-#define DMAR_OPERATION_TIMEOUT (HZ*60) /* 1m */
+#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) /* 10sec */
#define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
+
+static void flush_unmaps_timeout(unsigned long data);
+
+DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
+
+static struct intel_iommu *g_iommus;
+
+#define HIGH_WATER_MARK 250
+struct deferred_flush_tables {
+ int next;
+ struct iova *iova[HIGH_WATER_MARK];
+ struct dmar_domain *domain[HIGH_WATER_MARK];
+};
+
+static struct deferred_flush_tables *deferred_flush;
+
+/* bitmap for indexing intel_iommus */
+static int g_num_of_iommus;
+
+static DEFINE_SPINLOCK(async_umap_flush_lock);
+static LIST_HEAD(unmaps_to_do);
+
+static int timer_on;
+static long list_size;
+
static void domain_remove_dev_info(struct dmar_domain *domain);
static int dmar_disabled;
static int __initdata dmar_map_gfx = 1;
static int dmar_forcedac;
+static int intel_iommu_strict;
#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
static DEFINE_SPINLOCK(device_domain_lock);
printk(KERN_INFO
"Intel-IOMMU: disable GFX device mapping\n");
} else if (!strncmp(str, "forcedac", 8)) {
- printk (KERN_INFO
+ printk(KERN_INFO
"Intel-IOMMU: Forcing DAC for PCI devices\n");
dmar_forcedac = 1;
+ } else if (!strncmp(str, "strict", 6)) {
+ printk(KERN_INFO
+ "Intel-IOMMU: disable batched IOTLB flush\n");
+ intel_iommu_strict = 1;
}
str += strcspn(str, ",");
#define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \
{\
- unsigned long start_time = jiffies;\
+ cycles_t start_time = get_cycles();\
while (1) {\
sts = op (iommu->reg + offset);\
if (cond)\
break;\
- if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))\
+ if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start_time))\
panic("DMAR hardware is malfunctioning\n");\
cpu_relax();\
}\
non_present_entry_flush);
}
-static int iommu_get_alignment(u64 base, unsigned int size)
-{
- int t = 0;
- u64 end;
-
- end = base + size - 1;
- while (base != end) {
- t++;
- base >>= 1;
- end >>= 1;
- }
- return t;
-}
-
static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
u64 addr, unsigned int pages, int non_present_entry_flush)
{
- unsigned int align;
+ unsigned int mask;
BUG_ON(addr & (~PAGE_MASK_4K));
BUG_ON(pages == 0);
* PSI requires page size to be 2 ^ x, and the base address is naturally
* aligned to the size
*/
- align = iommu_get_alignment(addr >> PAGE_SHIFT_4K, pages);
+ mask = ilog2(__roundup_pow_of_two(pages));
/* Fallback to domain selective flush if size is too big */
- if (align > cap_max_amask_val(iommu->cap))
+ if (mask > cap_max_amask_val(iommu->cap))
return iommu_flush_iotlb_dsi(iommu, did,
non_present_entry_flush);
- addr >>= PAGE_SHIFT_4K + align;
- addr <<= PAGE_SHIFT_4K + align;
-
- return __iommu_flush_iotlb(iommu, did, addr, align,
+ return __iommu_flush_iotlb(iommu, did, addr, mask,
DMA_TLB_PSI_FLUSH, non_present_entry_flush);
}
+static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
+{
+ u32 pmen;
+ unsigned long flags;
+
+ spin_lock_irqsave(&iommu->register_lock, flags);
+ pmen = readl(iommu->reg + DMAR_PMEN_REG);
+ pmen &= ~DMA_PMEN_EPM;
+ writel(pmen, iommu->reg + DMAR_PMEN_REG);
+
+ /* wait for the protected region status bit to clear */
+ IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
+ readl, !(pmen & DMA_PMEN_PRS), pmen);
+
+ spin_unlock_irqrestore(&iommu->register_lock, flags);
+}
+
static int iommu_enable_translation(struct intel_iommu *iommu)
{
u32 sts;
/* iommu interrupt handling. Most stuff are MSI-like. */
-static char *fault_reason_strings[] =
+static const char *fault_reason_strings[] =
{
"Software",
"Present bit in root entry is clear",
"non-zero reserved fields in RTP",
"non-zero reserved fields in CTP",
"non-zero reserved fields in PTE",
- "Unknown"
};
-#define MAX_FAULT_REASON_IDX ARRAY_SIZE(fault_reason_strings)
+#define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
-char *dmar_get_fault_reason(u8 fault_reason)
+const char *dmar_get_fault_reason(u8 fault_reason)
{
if (fault_reason > MAX_FAULT_REASON_IDX)
- return fault_reason_strings[MAX_FAULT_REASON_IDX];
+ return "Unknown";
else
return fault_reason_strings[fault_reason];
}
static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
u8 fault_reason, u16 source_id, u64 addr)
{
- char *reason;
+ const char *reason;
reason = dmar_get_fault_reason(fault_reason);
set_bit(0, iommu->domain_ids);
return 0;
}
-
-static struct intel_iommu *alloc_iommu(struct dmar_drhd_unit *drhd)
+static struct intel_iommu *alloc_iommu(struct intel_iommu *iommu,
+ struct dmar_drhd_unit *drhd)
{
- struct intel_iommu *iommu;
int ret;
int map_size;
u32 ver;
- iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
- if (!iommu)
- return NULL;
iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
if (!iommu->reg) {
printk(KERN_ERR "IOMMU: can't map the region\n");
return iommu;
error_unmap:
iounmap(iommu->reg);
- iommu->reg = 0;
error:
kfree(iommu);
return NULL;
}
static struct iova_domain reserved_iova_list;
+static struct lock_class_key reserved_alloc_key;
+static struct lock_class_key reserved_rbtree_key;
static void dmar_init_reserved_ranges(void)
{
int i;
u64 addr, size;
- init_iova_domain(&reserved_iova_list);
+ init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
+
+ lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
+ &reserved_alloc_key);
+ lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
+ &reserved_rbtree_key);
/* IOAPIC ranges shouldn't be accessed by DMA */
iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
int adjust_width, agaw;
unsigned long sagaw;
- init_iova_domain(&domain->iovad);
+ init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
spin_lock_init(&domain->mapping_lock);
domain_reserve_special_ranges(domain);
list_del(&info->link);
list_del(&info->global);
if (info->dev)
- info->dev->sysdata = NULL;
+ info->dev->dev.archdata.iommu = NULL;
spin_unlock_irqrestore(&device_domain_lock, flags);
detach_domain_for_dev(info->domain, info->bus, info->devfn);
/*
* find_domain
- * Note: we use struct pci_dev->sysdata stores the info
+ * Note: we use struct pci_dev->dev.archdata.iommu stores the info
*/
struct dmar_domain *
find_domain(struct pci_dev *pdev)
struct device_domain_info *info;
/* No lock here, assumes no domain exit in normal case */
- info = pdev->sysdata;
+ info = pdev->dev.archdata.iommu;
if (info)
return info->domain;
return NULL;
int index;
while (dev) {
- for (index = 0; index < cnt; index ++)
+ for (index = 0; index < cnt; index++)
if (dev == devices[index])
return 1;
}
list_add(&info->link, &domain->devices);
list_add(&info->global, &device_domain_list);
- pdev->sysdata = info;
+ pdev->dev.archdata.iommu = info;
spin_unlock_irqrestore(&device_domain_lock, flags);
return domain;
error:
static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
struct pci_dev *pdev)
{
- if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
+ if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
return 0;
return iommu_prepare_identity_map(pdev, rmrr->base_address,
rmrr->end_address + 1);
}
#ifdef CONFIG_DMAR_GFX_WA
-extern int arch_get_ram_range(int slot, u64 *addr, u64 *size);
+struct iommu_prepare_data {
+ struct pci_dev *pdev;
+ int ret;
+};
+
+static int __init iommu_prepare_work_fn(unsigned long start_pfn,
+ unsigned long end_pfn, void *datax)
+{
+ struct iommu_prepare_data *data;
+
+ data = (struct iommu_prepare_data *)datax;
+
+ data->ret = iommu_prepare_identity_map(data->pdev,
+ start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
+ return data->ret;
+
+}
+
+static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
+{
+ int nid;
+ struct iommu_prepare_data data;
+
+ data.pdev = pdev;
+ data.ret = 0;
+
+ for_each_online_node(nid) {
+ work_with_active_regions(nid, iommu_prepare_work_fn, &data);
+ if (data.ret)
+ return data.ret;
+ }
+ return data.ret;
+}
+
static void __init iommu_prepare_gfx_mapping(void)
{
struct pci_dev *pdev = NULL;
- u64 base, size;
- int slot;
int ret;
for_each_pci_dev(pdev) {
- if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO ||
+ if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
!IS_GFX_DEVICE(pdev))
continue;
printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
pci_name(pdev));
- slot = arch_get_ram_range(0, &base, &size);
- while (slot >= 0) {
- ret = iommu_prepare_identity_map(pdev,
- base, base + size);
- if (ret)
- goto error;
- slot = arch_get_ram_range(slot, &base, &size);
- }
- continue;
-error:
- printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
+ ret = iommu_prepare_with_active_regions(pdev);
+ if (ret)
+ printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
}
}
#endif
struct dmar_rmrr_unit *rmrr;
struct pci_dev *pdev;
struct intel_iommu *iommu;
- int ret, unit = 0;
+ int i, ret, unit = 0;
/*
* for each drhd
for_each_drhd_unit(drhd) {
if (drhd->ignored)
continue;
- iommu = alloc_iommu(drhd);
+ g_num_of_iommus++;
+ /*
+ * lock not needed as this is only incremented in the single
+ * threaded kernel __init code path all other access are read
+ * only
+ */
+ }
+
+ g_iommus = kzalloc(g_num_of_iommus * sizeof(*iommu), GFP_KERNEL);
+ if (!g_iommus) {
+ ret = -ENOMEM;
+ goto error;
+ }
+
+ deferred_flush = kzalloc(g_num_of_iommus *
+ sizeof(struct deferred_flush_tables), GFP_KERNEL);
+ if (!deferred_flush) {
+ ret = -ENOMEM;
+ goto error;
+ }
+
+ i = 0;
+ for_each_drhd_unit(drhd) {
+ if (drhd->ignored)
+ continue;
+ iommu = alloc_iommu(&g_iommus[i], drhd);
+ i++;
if (!iommu) {
ret = -ENOMEM;
goto error;
* endfor
*/
for_each_rmrr_units(rmrr) {
- int i;
for (i = 0; i < rmrr->devices_cnt; i++) {
pdev = rmrr->devices[i];
/* some BIOS lists non-exist devices in DMAR table */
iommu_flush_context_global(iommu, 0);
iommu_flush_iotlb_global(iommu, 0);
+ iommu_disable_protect_mem_regions(iommu);
+
ret = iommu_enable_translation(iommu);
if (ret)
goto error;
iommu = drhd->iommu;
free_iommu(iommu);
}
+ kfree(g_iommus);
return ret;
}
}
struct iova *
-iommu_alloc_iova(struct dmar_domain *domain, void *host_addr, size_t size,
- u64 start, u64 end)
+iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
{
- u64 start_addr;
struct iova *piova;
/* Make sure it's in range */
- if ((start > DOMAIN_MAX_ADDR(domain->gaw)) || end < start)
- return NULL;
-
end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
- start_addr = PAGE_ALIGN_4K(start);
- size = aligned_size((u64)host_addr, size);
- if (!size || (start_addr + size > end))
+ if (!size || (IOVA_START_ADDR + size > end))
return NULL;
piova = alloc_iova(&domain->iovad,
- size >> PAGE_SHIFT_4K, IOVA_PFN(end));
-
+ size >> PAGE_SHIFT_4K, IOVA_PFN(end), 1);
return piova;
}
-static dma_addr_t __intel_map_single(struct device *dev, void *addr,
- size_t size, int dir, u64 *flush_addr, unsigned int *flush_size)
+static struct iova *
+__intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
+ size_t size)
{
- struct dmar_domain *domain;
struct pci_dev *pdev = to_pci_dev(dev);
- int ret;
- int prot = 0;
struct iova *iova = NULL;
- u64 start_addr;
-
- addr = (void *)virt_to_phys(addr);
-
- domain = get_domain_for_dev(pdev,
- DEFAULT_DOMAIN_ADDRESS_WIDTH);
- if (!domain) {
- printk(KERN_ERR
- "Allocating domain for %s failed", pci_name(pdev));
- return 0;
- }
-
- start_addr = IOVA_START_ADDR;
if ((pdev->dma_mask <= DMA_32BIT_MASK) || (dmar_forcedac)) {
- iova = iommu_alloc_iova(domain, addr, size, start_addr,
- pdev->dma_mask);
+ iova = iommu_alloc_iova(domain, size, pdev->dma_mask);
} else {
/*
* First try to allocate an io virtual address in
* DMA_32BIT_MASK and if that fails then try allocating
- * from higer range
+ * from higher range
*/
- iova = iommu_alloc_iova(domain, addr, size, start_addr,
- DMA_32BIT_MASK);
+ iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
if (!iova)
- iova = iommu_alloc_iova(domain, addr, size, start_addr,
- pdev->dma_mask);
+ iova = iommu_alloc_iova(domain, size, pdev->dma_mask);
}
if (!iova) {
printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
- return 0;
+ return NULL;
+ }
+
+ return iova;
+}
+
+static struct dmar_domain *
+get_valid_domain_for_dev(struct pci_dev *pdev)
+{
+ struct dmar_domain *domain;
+ int ret;
+
+ domain = get_domain_for_dev(pdev,
+ DEFAULT_DOMAIN_ADDRESS_WIDTH);
+ if (!domain) {
+ printk(KERN_ERR
+ "Allocating domain for %s failed", pci_name(pdev));
+ return NULL;
}
/* make sure context mapping is ok */
if (unlikely(!domain_context_mapped(domain, pdev))) {
ret = domain_context_mapping(domain, pdev);
- if (ret)
- goto error;
+ if (ret) {
+ printk(KERN_ERR
+ "Domain context map for %s failed",
+ pci_name(pdev));
+ return NULL;
+ }
}
+ return domain;
+}
+
+static dma_addr_t
+intel_map_single(struct device *hwdev, phys_addr_t paddr, size_t size, int dir)
+{
+ struct pci_dev *pdev = to_pci_dev(hwdev);
+ struct dmar_domain *domain;
+ unsigned long start_paddr;
+ struct iova *iova;
+ int prot = 0;
+ int ret;
+
+ BUG_ON(dir == DMA_NONE);
+ if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
+ return paddr;
+
+ domain = get_valid_domain_for_dev(pdev);
+ if (!domain)
+ return 0;
+
+ size = aligned_size((u64)paddr, size);
+
+ iova = __intel_alloc_iova(hwdev, domain, size);
+ if (!iova)
+ goto error;
+
+ start_paddr = iova->pfn_lo << PAGE_SHIFT_4K;
+
/*
* Check if DMAR supports zero-length reads on write only
* mappings..
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
prot |= DMA_PTE_WRITE;
/*
- * addr - (addr + size) might be partial page, we should map the whole
+ * paddr - (paddr + size) might be partial page, we should map the whole
* page. Note: if two part of one page are separately mapped, we
- * might have two guest_addr mapping to the same host addr, but this
+ * might have two guest_addr mapping to the same host paddr, but this
* is not a big problem
*/
- ret = domain_page_mapping(domain, iova->pfn_lo << PAGE_SHIFT_4K,
- ((u64)addr) & PAGE_MASK_4K,
- (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K, prot);
+ ret = domain_page_mapping(domain, start_paddr,
+ ((u64)paddr) & PAGE_MASK_4K, size, prot);
if (ret)
goto error;
pr_debug("Device %s request: %lx@%llx mapping: %lx@%llx, dir %d\n",
- pci_name(pdev), size, (u64)addr,
- (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K,
- (u64)(iova->pfn_lo << PAGE_SHIFT_4K), dir);
+ pci_name(pdev), size, (u64)paddr,
+ size, (u64)start_paddr, dir);
+
+ /* it's a non-present to present mapping */
+ ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
+ start_paddr, size >> PAGE_SHIFT_4K, 1);
+ if (ret)
+ iommu_flush_write_buffer(domain->iommu);
+
+ return (start_paddr + ((u64)paddr & (~PAGE_MASK_4K)));
- *flush_addr = iova->pfn_lo << PAGE_SHIFT_4K;
- *flush_size = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K;
- return (iova->pfn_lo << PAGE_SHIFT_4K) + ((u64)addr & (~PAGE_MASK_4K));
error:
- __free_iova(&domain->iovad, iova);
+ if (iova)
+ __free_iova(&domain->iovad, iova);
printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
- pci_name(pdev), size, (u64)addr, dir);
+ pci_name(pdev), size, (u64)paddr, dir);
return 0;
}
-static dma_addr_t intel_map_single(struct device *hwdev, void *addr,
- size_t size, int dir)
+static void flush_unmaps(void)
{
- struct pci_dev *pdev = to_pci_dev(hwdev);
- dma_addr_t ret;
- struct dmar_domain *domain;
- u64 flush_addr;
- unsigned int flush_size;
+ int i, j;
- BUG_ON(dir == DMA_NONE);
- if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
- return virt_to_bus(addr);
+ timer_on = 0;
- ret = __intel_map_single(hwdev, addr, size,
- dir, &flush_addr, &flush_size);
- if (ret) {
- domain = find_domain(pdev);
- /* it's a non-present to present mapping */
- if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
- flush_addr, flush_size >> PAGE_SHIFT_4K, 1))
- iommu_flush_write_buffer(domain->iommu);
+ /* just flush them all */
+ for (i = 0; i < g_num_of_iommus; i++) {
+ if (deferred_flush[i].next) {
+ iommu_flush_iotlb_global(&g_iommus[i], 0);
+ for (j = 0; j < deferred_flush[i].next; j++) {
+ __free_iova(&deferred_flush[i].domain[j]->iovad,
+ deferred_flush[i].iova[j]);
+ }
+ deferred_flush[i].next = 0;
+ }
}
- return ret;
+
+ list_size = 0;
}
-static void __intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
- size_t size, int dir, u64 *flush_addr, unsigned int *flush_size)
+static void flush_unmaps_timeout(unsigned long data)
{
- struct dmar_domain *domain;
- struct pci_dev *pdev = to_pci_dev(dev);
- struct iova *iova;
+ unsigned long flags;
- domain = find_domain(pdev);
- BUG_ON(!domain);
+ spin_lock_irqsave(&async_umap_flush_lock, flags);
+ flush_unmaps();
+ spin_unlock_irqrestore(&async_umap_flush_lock, flags);
+}
- iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
- if (!iova) {
- *flush_size = 0;
- return;
+static void add_unmap(struct dmar_domain *dom, struct iova *iova)
+{
+ unsigned long flags;
+ int next, iommu_id;
+
+ spin_lock_irqsave(&async_umap_flush_lock, flags);
+ if (list_size == HIGH_WATER_MARK)
+ flush_unmaps();
+
+ iommu_id = dom->iommu - g_iommus;
+ next = deferred_flush[iommu_id].next;
+ deferred_flush[iommu_id].domain[next] = dom;
+ deferred_flush[iommu_id].iova[next] = iova;
+ deferred_flush[iommu_id].next++;
+
+ if (!timer_on) {
+ mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
+ timer_on = 1;
}
- pr_debug("Device %s unmapping: %lx@%llx\n",
- pci_name(pdev),
- (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K,
- (u64)(iova->pfn_lo << PAGE_SHIFT_4K));
-
- *flush_addr = iova->pfn_lo << PAGE_SHIFT_4K;
- *flush_size = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K;
- /* clear the whole page, not just dev_addr - (dev_addr + size) */
- dma_pte_clear_range(domain, *flush_addr, *flush_addr + *flush_size);
- /* free page tables */
- dma_pte_free_pagetable(domain, *flush_addr, *flush_addr + *flush_size);
- /* free iova */
- __free_iova(&domain->iovad, iova);
+ list_size++;
+ spin_unlock_irqrestore(&async_umap_flush_lock, flags);
}
static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
{
struct pci_dev *pdev = to_pci_dev(dev);
struct dmar_domain *domain;
- u64 flush_addr;
- unsigned int flush_size;
+ unsigned long start_addr;
+ struct iova *iova;
- if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
+ if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
return;
-
domain = find_domain(pdev);
- __intel_unmap_single(dev, dev_addr, size,
- dir, &flush_addr, &flush_size);
- if (flush_size == 0)
+ BUG_ON(!domain);
+
+ iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
+ if (!iova)
return;
- if (iommu_flush_iotlb_psi(domain->iommu, domain->id, flush_addr,
- flush_size >> PAGE_SHIFT_4K, 0))
- iommu_flush_write_buffer(domain->iommu);
+
+ start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
+ size = aligned_size((u64)dev_addr, size);
+
+ pr_debug("Device %s unmapping: %lx@%llx\n",
+ pci_name(pdev), size, (u64)start_addr);
+
+ /* clear the whole page */
+ dma_pte_clear_range(domain, start_addr, start_addr + size);
+ /* free page tables */
+ dma_pte_free_pagetable(domain, start_addr, start_addr + size);
+ if (intel_iommu_strict) {
+ if (iommu_flush_iotlb_psi(domain->iommu,
+ domain->id, start_addr, size >> PAGE_SHIFT_4K, 0))
+ iommu_flush_write_buffer(domain->iommu);
+ /* free iova */
+ __free_iova(&domain->iovad, iova);
+ } else {
+ add_unmap(domain, iova);
+ /*
+ * queue up the release of the unmap to save the 1/6th of the
+ * cpu used up by the iotlb flush operation...
+ */
+ }
}
static void * intel_alloc_coherent(struct device *hwdev, size_t size,
return NULL;
memset(vaddr, 0, size);
- *dma_handle = intel_map_single(hwdev, vaddr, size, DMA_BIDIRECTIONAL);
+ *dma_handle = intel_map_single(hwdev, virt_to_bus(vaddr), size, DMA_BIDIRECTIONAL);
if (*dma_handle)
return vaddr;
free_pages((unsigned long)vaddr, order);
free_pages((unsigned long)vaddr, order);
}
-static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sg,
+#define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
+static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
int nelems, int dir)
{
int i;
struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
- u64 flush_addr;
- unsigned int flush_size;
+ unsigned long start_addr;
+ struct iova *iova;
+ size_t size = 0;
+ void *addr;
+ struct scatterlist *sg;
- if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
+ if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
return;
domain = find_domain(pdev);
- for (i = 0; i < nelems; i++, sg++)
- __intel_unmap_single(hwdev, sg->dma_address,
- sg->dma_length, dir, &flush_addr, &flush_size);
- if (iommu_flush_iotlb_dsi(domain->iommu, domain->id, 0))
+ iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
+ if (!iova)
+ return;
+ for_each_sg(sglist, sg, nelems, i) {
+ addr = SG_ENT_VIRT_ADDRESS(sg);
+ size += aligned_size((u64)addr, sg->length);
+ }
+
+ start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
+
+ /* clear the whole page */
+ dma_pte_clear_range(domain, start_addr, start_addr + size);
+ /* free page tables */
+ dma_pte_free_pagetable(domain, start_addr, start_addr + size);
+
+ if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
+ size >> PAGE_SHIFT_4K, 0))
iommu_flush_write_buffer(domain->iommu);
+
+ /* free iova */
+ __free_iova(&domain->iovad, iova);
}
-#define SG_ENT_VIRT_ADDRESS(sg) (page_address((sg)->page) + (sg)->offset)
static int intel_nontranslate_map_sg(struct device *hddev,
- struct scatterlist *sg, int nelems, int dir)
+ struct scatterlist *sglist, int nelems, int dir)
{
int i;
+ struct scatterlist *sg;
- for (i = 0; i < nelems; i++) {
- struct scatterlist *s = &sg[i];
- BUG_ON(!s->page);
- s->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(s));
- s->dma_length = s->length;
+ for_each_sg(sglist, sg, nelems, i) {
+ BUG_ON(!sg_page(sg));
+ sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
+ sg->dma_length = sg->length;
}
return nelems;
}
-static int intel_map_sg(struct device *hwdev, struct scatterlist *sg,
- int nelems, int dir)
+static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist,
+ int nelems, int dir)
{
void *addr;
int i;
- dma_addr_t dma_handle;
struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
- u64 flush_addr;
- unsigned int flush_size;
+ size_t size = 0;
+ int prot = 0;
+ size_t offset = 0;
+ struct iova *iova = NULL;
+ int ret;
+ struct scatterlist *sg;
+ unsigned long start_addr;
BUG_ON(dir == DMA_NONE);
- if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
- return intel_nontranslate_map_sg(hwdev, sg, nelems, dir);
+ if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
+ return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
+
+ domain = get_valid_domain_for_dev(pdev);
+ if (!domain)
+ return 0;
- for (i = 0; i < nelems; i++, sg++) {
+ for_each_sg(sglist, sg, nelems, i) {
addr = SG_ENT_VIRT_ADDRESS(sg);
- dma_handle = __intel_map_single(hwdev, addr,
- sg->length, dir, &flush_addr, &flush_size);
- if (!dma_handle) {
- intel_unmap_sg(hwdev, sg - i, i, dir);
- sg[0].dma_length = 0;
+ addr = (void *)virt_to_phys(addr);
+ size += aligned_size((u64)addr, sg->length);
+ }
+
+ iova = __intel_alloc_iova(hwdev, domain, size);
+ if (!iova) {
+ sglist->dma_length = 0;
+ return 0;
+ }
+
+ /*
+ * Check if DMAR supports zero-length reads on write only
+ * mappings..
+ */
+ if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
+ !cap_zlr(domain->iommu->cap))
+ prot |= DMA_PTE_READ;
+ if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
+ prot |= DMA_PTE_WRITE;
+
+ start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
+ offset = 0;
+ for_each_sg(sglist, sg, nelems, i) {
+ addr = SG_ENT_VIRT_ADDRESS(sg);
+ addr = (void *)virt_to_phys(addr);
+ size = aligned_size((u64)addr, sg->length);
+ ret = domain_page_mapping(domain, start_addr + offset,
+ ((u64)addr) & PAGE_MASK_4K,
+ size, prot);
+ if (ret) {
+ /* clear the page */
+ dma_pte_clear_range(domain, start_addr,
+ start_addr + offset);
+ /* free page tables */
+ dma_pte_free_pagetable(domain, start_addr,
+ start_addr + offset);
+ /* free iova */
+ __free_iova(&domain->iovad, iova);
return 0;
}
- sg->dma_address = dma_handle;
+ sg->dma_address = start_addr + offset +
+ ((u64)addr & (~PAGE_MASK_4K));
sg->dma_length = sg->length;
+ offset += size;
}
- domain = find_domain(pdev);
-
/* it's a non-present to present mapping */
- if (iommu_flush_iotlb_dsi(domain->iommu, domain->id, 1))
+ if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
+ start_addr, offset >> PAGE_SHIFT_4K, 1))
iommu_flush_write_buffer(domain->iommu);
return nelems;
}
for (i = 0; i < drhd->devices_cnt; i++) {
if (!drhd->devices[i])
continue;
- drhd->devices[i]->sysdata = DUMMY_DEVICE_DOMAIN_INFO;
+ drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
}
}
}
printk(KERN_INFO
"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
+ init_timer(&unmap_timer);
force_iommu = 1;
dma_ops = &intel_dma_ops;
return 0;
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff