2 * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/pci.h>
21 #include <linux/gfp.h>
22 #include <linux/bitops.h>
23 #include <linux/debugfs.h>
24 #include <linux/scatterlist.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/iommu-helper.h>
27 #include <linux/iommu.h>
28 #include <asm/proto.h>
29 #include <asm/iommu.h>
31 #include <asm/amd_iommu_proto.h>
32 #include <asm/amd_iommu_types.h>
33 #include <asm/amd_iommu.h>
35 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
37 #define EXIT_LOOP_COUNT 10000000
39 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
41 /* A list of preallocated protection domains */
42 static LIST_HEAD(iommu_pd_list);
43 static DEFINE_SPINLOCK(iommu_pd_list_lock);
46 * Domain for untranslated devices - only allocated
47 * if iommu=pt passed on kernel cmd line.
49 static struct protection_domain *pt_domain;
51 static struct iommu_ops amd_iommu_ops;
54 * general struct to manage commands send to an IOMMU
60 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
61 struct unity_map_entry *e);
62 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
63 unsigned long start_page,
65 static void reset_iommu_command_buffer(struct amd_iommu *iommu);
66 static void update_domain(struct protection_domain *domain);
68 /****************************************************************************
72 ****************************************************************************/
74 static inline u16 get_device_id(struct device *dev)
76 struct pci_dev *pdev = to_pci_dev(dev);
78 return calc_devid(pdev->bus->number, pdev->devfn);
82 * In this function the list of preallocated protection domains is traversed to
83 * find the domain for a specific device
85 static struct dma_ops_domain *find_protection_domain(u16 devid)
87 struct dma_ops_domain *entry, *ret = NULL;
89 u16 alias = amd_iommu_alias_table[devid];
91 if (list_empty(&iommu_pd_list))
94 spin_lock_irqsave(&iommu_pd_list_lock, flags);
96 list_for_each_entry(entry, &iommu_pd_list, list) {
97 if (entry->target_dev == devid ||
98 entry->target_dev == alias) {
104 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
110 * This function checks if the driver got a valid device from the caller to
111 * avoid dereferencing invalid pointers.
113 static bool check_device(struct device *dev)
117 if (!dev || !dev->dma_mask)
120 /* No device or no PCI device */
121 if (!dev || dev->bus != &pci_bus_type)
124 devid = get_device_id(dev);
126 /* Out of our scope? */
127 if (devid > amd_iommu_last_bdf)
130 if (amd_iommu_rlookup_table[devid] == NULL)
136 #ifdef CONFIG_AMD_IOMMU_STATS
139 * Initialization code for statistics collection
142 DECLARE_STATS_COUNTER(compl_wait);
143 DECLARE_STATS_COUNTER(cnt_map_single);
144 DECLARE_STATS_COUNTER(cnt_unmap_single);
145 DECLARE_STATS_COUNTER(cnt_map_sg);
146 DECLARE_STATS_COUNTER(cnt_unmap_sg);
147 DECLARE_STATS_COUNTER(cnt_alloc_coherent);
148 DECLARE_STATS_COUNTER(cnt_free_coherent);
149 DECLARE_STATS_COUNTER(cross_page);
150 DECLARE_STATS_COUNTER(domain_flush_single);
151 DECLARE_STATS_COUNTER(domain_flush_all);
152 DECLARE_STATS_COUNTER(alloced_io_mem);
153 DECLARE_STATS_COUNTER(total_map_requests);
155 static struct dentry *stats_dir;
156 static struct dentry *de_isolate;
157 static struct dentry *de_fflush;
159 static void amd_iommu_stats_add(struct __iommu_counter *cnt)
161 if (stats_dir == NULL)
164 cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
168 static void amd_iommu_stats_init(void)
170 stats_dir = debugfs_create_dir("amd-iommu", NULL);
171 if (stats_dir == NULL)
174 de_isolate = debugfs_create_bool("isolation", 0444, stats_dir,
175 (u32 *)&amd_iommu_isolate);
177 de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
178 (u32 *)&amd_iommu_unmap_flush);
180 amd_iommu_stats_add(&compl_wait);
181 amd_iommu_stats_add(&cnt_map_single);
182 amd_iommu_stats_add(&cnt_unmap_single);
183 amd_iommu_stats_add(&cnt_map_sg);
184 amd_iommu_stats_add(&cnt_unmap_sg);
185 amd_iommu_stats_add(&cnt_alloc_coherent);
186 amd_iommu_stats_add(&cnt_free_coherent);
187 amd_iommu_stats_add(&cross_page);
188 amd_iommu_stats_add(&domain_flush_single);
189 amd_iommu_stats_add(&domain_flush_all);
190 amd_iommu_stats_add(&alloced_io_mem);
191 amd_iommu_stats_add(&total_map_requests);
196 /****************************************************************************
198 * Interrupt handling functions
200 ****************************************************************************/
202 static void dump_dte_entry(u16 devid)
206 for (i = 0; i < 8; ++i)
207 pr_err("AMD-Vi: DTE[%d]: %08x\n", i,
208 amd_iommu_dev_table[devid].data[i]);
211 static void dump_command(unsigned long phys_addr)
213 struct iommu_cmd *cmd = phys_to_virt(phys_addr);
216 for (i = 0; i < 4; ++i)
217 pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
220 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
223 int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
224 int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
225 int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
226 int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
227 u64 address = (u64)(((u64)event[3]) << 32) | event[2];
229 printk(KERN_ERR "AMD-Vi: Event logged [");
232 case EVENT_TYPE_ILL_DEV:
233 printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
234 "address=0x%016llx flags=0x%04x]\n",
235 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
237 dump_dte_entry(devid);
239 case EVENT_TYPE_IO_FAULT:
240 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
241 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
242 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
243 domid, address, flags);
245 case EVENT_TYPE_DEV_TAB_ERR:
246 printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
247 "address=0x%016llx flags=0x%04x]\n",
248 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
251 case EVENT_TYPE_PAGE_TAB_ERR:
252 printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
253 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
254 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
255 domid, address, flags);
257 case EVENT_TYPE_ILL_CMD:
258 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
259 reset_iommu_command_buffer(iommu);
260 dump_command(address);
262 case EVENT_TYPE_CMD_HARD_ERR:
263 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
264 "flags=0x%04x]\n", address, flags);
266 case EVENT_TYPE_IOTLB_INV_TO:
267 printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
268 "address=0x%016llx]\n",
269 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
272 case EVENT_TYPE_INV_DEV_REQ:
273 printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
274 "address=0x%016llx flags=0x%04x]\n",
275 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
279 printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
283 static void iommu_poll_events(struct amd_iommu *iommu)
288 spin_lock_irqsave(&iommu->lock, flags);
290 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
291 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
293 while (head != tail) {
294 iommu_print_event(iommu, iommu->evt_buf + head);
295 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
298 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
300 spin_unlock_irqrestore(&iommu->lock, flags);
303 irqreturn_t amd_iommu_int_handler(int irq, void *data)
305 struct amd_iommu *iommu;
307 for_each_iommu(iommu)
308 iommu_poll_events(iommu);
313 /****************************************************************************
315 * IOMMU command queuing functions
317 ****************************************************************************/
320 * Writes the command to the IOMMUs command buffer and informs the
321 * hardware about the new command. Must be called with iommu->lock held.
323 static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
328 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
329 target = iommu->cmd_buf + tail;
330 memcpy_toio(target, cmd, sizeof(*cmd));
331 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
332 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
335 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
341 * General queuing function for commands. Takes iommu->lock and calls
342 * __iommu_queue_command().
344 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
349 spin_lock_irqsave(&iommu->lock, flags);
350 ret = __iommu_queue_command(iommu, cmd);
352 iommu->need_sync = true;
353 spin_unlock_irqrestore(&iommu->lock, flags);
359 * This function waits until an IOMMU has completed a completion
362 static void __iommu_wait_for_completion(struct amd_iommu *iommu)
368 INC_STATS_COUNTER(compl_wait);
370 while (!ready && (i < EXIT_LOOP_COUNT)) {
372 /* wait for the bit to become one */
373 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
374 ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
377 /* set bit back to zero */
378 status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
379 writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
381 if (unlikely(i == EXIT_LOOP_COUNT)) {
382 spin_unlock(&iommu->lock);
383 reset_iommu_command_buffer(iommu);
384 spin_lock(&iommu->lock);
389 * This function queues a completion wait command into the command
392 static int __iommu_completion_wait(struct amd_iommu *iommu)
394 struct iommu_cmd cmd;
396 memset(&cmd, 0, sizeof(cmd));
397 cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
398 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
400 return __iommu_queue_command(iommu, &cmd);
404 * This function is called whenever we need to ensure that the IOMMU has
405 * completed execution of all commands we sent. It sends a
406 * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
407 * us about that by writing a value to a physical address we pass with
410 static int iommu_completion_wait(struct amd_iommu *iommu)
415 spin_lock_irqsave(&iommu->lock, flags);
417 if (!iommu->need_sync)
420 ret = __iommu_completion_wait(iommu);
422 iommu->need_sync = false;
427 __iommu_wait_for_completion(iommu);
430 spin_unlock_irqrestore(&iommu->lock, flags);
435 static void iommu_flush_complete(struct protection_domain *domain)
439 for (i = 0; i < amd_iommus_present; ++i) {
440 if (!domain->dev_iommu[i])
444 * Devices of this domain are behind this IOMMU
445 * We need to wait for completion of all commands.
447 iommu_completion_wait(amd_iommus[i]);
452 * Command send function for invalidating a device table entry
454 static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
456 struct iommu_cmd cmd;
459 BUG_ON(iommu == NULL);
461 memset(&cmd, 0, sizeof(cmd));
462 CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
465 ret = iommu_queue_command(iommu, &cmd);
470 static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
471 u16 domid, int pde, int s)
473 memset(cmd, 0, sizeof(*cmd));
474 address &= PAGE_MASK;
475 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
476 cmd->data[1] |= domid;
477 cmd->data[2] = lower_32_bits(address);
478 cmd->data[3] = upper_32_bits(address);
479 if (s) /* size bit - we flush more than one 4kb page */
480 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
481 if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
482 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
486 * Generic command send function for invalidaing TLB entries
488 static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
489 u64 address, u16 domid, int pde, int s)
491 struct iommu_cmd cmd;
494 __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
496 ret = iommu_queue_command(iommu, &cmd);
502 * TLB invalidation function which is called from the mapping functions.
503 * It invalidates a single PTE if the range to flush is within a single
504 * page. Otherwise it flushes the whole TLB of the IOMMU.
506 static void __iommu_flush_pages(struct protection_domain *domain,
507 u64 address, size_t size, int pde)
510 unsigned long pages = iommu_num_pages(address, size, PAGE_SIZE);
512 address &= PAGE_MASK;
516 * If we have to flush more than one page, flush all
517 * TLB entries for this domain
519 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
524 for (i = 0; i < amd_iommus_present; ++i) {
525 if (!domain->dev_iommu[i])
529 * Devices of this domain are behind this IOMMU
530 * We need a TLB flush
532 iommu_queue_inv_iommu_pages(amd_iommus[i], address,
539 static void iommu_flush_pages(struct protection_domain *domain,
540 u64 address, size_t size)
542 __iommu_flush_pages(domain, address, size, 0);
545 /* Flush the whole IO/TLB for a given protection domain */
546 static void iommu_flush_tlb(struct protection_domain *domain)
548 __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
551 /* Flush the whole IO/TLB for a given protection domain - including PDE */
552 static void iommu_flush_tlb_pde(struct protection_domain *domain)
554 __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
558 * This function flushes all domains that have devices on the given IOMMU
560 static void flush_all_domains_on_iommu(struct amd_iommu *iommu)
562 u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
563 struct protection_domain *domain;
566 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
568 list_for_each_entry(domain, &amd_iommu_pd_list, list) {
569 if (domain->dev_iommu[iommu->index] == 0)
572 spin_lock(&domain->lock);
573 iommu_queue_inv_iommu_pages(iommu, address, domain->id, 1, 1);
574 iommu_flush_complete(domain);
575 spin_unlock(&domain->lock);
578 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
582 * This function uses heavy locking and may disable irqs for some time. But
583 * this is no issue because it is only called during resume.
585 void amd_iommu_flush_all_domains(void)
587 struct protection_domain *domain;
590 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
592 list_for_each_entry(domain, &amd_iommu_pd_list, list) {
593 spin_lock(&domain->lock);
594 iommu_flush_tlb_pde(domain);
595 iommu_flush_complete(domain);
596 spin_unlock(&domain->lock);
599 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
602 static void flush_all_devices_for_iommu(struct amd_iommu *iommu)
606 for (i = 0; i <= amd_iommu_last_bdf; ++i) {
607 if (iommu != amd_iommu_rlookup_table[i])
610 iommu_queue_inv_dev_entry(iommu, i);
611 iommu_completion_wait(iommu);
615 static void flush_devices_by_domain(struct protection_domain *domain)
617 struct amd_iommu *iommu;
620 for (i = 0; i <= amd_iommu_last_bdf; ++i) {
621 if ((domain == NULL && amd_iommu_pd_table[i] == NULL) ||
622 (amd_iommu_pd_table[i] != domain))
625 iommu = amd_iommu_rlookup_table[i];
629 iommu_queue_inv_dev_entry(iommu, i);
630 iommu_completion_wait(iommu);
634 static void reset_iommu_command_buffer(struct amd_iommu *iommu)
636 pr_err("AMD-Vi: Resetting IOMMU command buffer\n");
638 if (iommu->reset_in_progress)
639 panic("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n");
641 iommu->reset_in_progress = true;
643 amd_iommu_reset_cmd_buffer(iommu);
644 flush_all_devices_for_iommu(iommu);
645 flush_all_domains_on_iommu(iommu);
647 iommu->reset_in_progress = false;
650 void amd_iommu_flush_all_devices(void)
652 flush_devices_by_domain(NULL);
655 /****************************************************************************
657 * The functions below are used the create the page table mappings for
658 * unity mapped regions.
660 ****************************************************************************/
663 * This function is used to add another level to an IO page table. Adding
664 * another level increases the size of the address space by 9 bits to a size up
667 static bool increase_address_space(struct protection_domain *domain,
672 if (domain->mode == PAGE_MODE_6_LEVEL)
673 /* address space already 64 bit large */
676 pte = (void *)get_zeroed_page(gfp);
680 *pte = PM_LEVEL_PDE(domain->mode,
681 virt_to_phys(domain->pt_root));
682 domain->pt_root = pte;
684 domain->updated = true;
689 static u64 *alloc_pte(struct protection_domain *domain,
690 unsigned long address,
698 while (address > PM_LEVEL_SIZE(domain->mode))
699 increase_address_space(domain, gfp);
701 level = domain->mode - 1;
702 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
704 while (level > end_lvl) {
705 if (!IOMMU_PTE_PRESENT(*pte)) {
706 page = (u64 *)get_zeroed_page(gfp);
709 *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
714 pte = IOMMU_PTE_PAGE(*pte);
716 if (pte_page && level == end_lvl)
719 pte = &pte[PM_LEVEL_INDEX(level, address)];
726 * This function checks if there is a PTE for a given dma address. If
727 * there is one, it returns the pointer to it.
729 static u64 *fetch_pte(struct protection_domain *domain,
730 unsigned long address, int map_size)
735 level = domain->mode - 1;
736 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
738 while (level > map_size) {
739 if (!IOMMU_PTE_PRESENT(*pte))
744 pte = IOMMU_PTE_PAGE(*pte);
745 pte = &pte[PM_LEVEL_INDEX(level, address)];
747 if ((PM_PTE_LEVEL(*pte) == 0) && level != map_size) {
757 * Generic mapping functions. It maps a physical address into a DMA
758 * address space. It allocates the page table pages if necessary.
759 * In the future it can be extended to a generic mapping function
760 * supporting all features of AMD IOMMU page tables like level skipping
761 * and full 64 bit address spaces.
763 static int iommu_map_page(struct protection_domain *dom,
764 unsigned long bus_addr,
765 unsigned long phys_addr,
771 bus_addr = PAGE_ALIGN(bus_addr);
772 phys_addr = PAGE_ALIGN(phys_addr);
774 BUG_ON(!PM_ALIGNED(map_size, bus_addr));
775 BUG_ON(!PM_ALIGNED(map_size, phys_addr));
777 if (!(prot & IOMMU_PROT_MASK))
780 pte = alloc_pte(dom, bus_addr, map_size, NULL, GFP_KERNEL);
782 if (IOMMU_PTE_PRESENT(*pte))
785 __pte = phys_addr | IOMMU_PTE_P;
786 if (prot & IOMMU_PROT_IR)
787 __pte |= IOMMU_PTE_IR;
788 if (prot & IOMMU_PROT_IW)
789 __pte |= IOMMU_PTE_IW;
798 static void iommu_unmap_page(struct protection_domain *dom,
799 unsigned long bus_addr, int map_size)
801 u64 *pte = fetch_pte(dom, bus_addr, map_size);
808 * This function checks if a specific unity mapping entry is needed for
809 * this specific IOMMU.
811 static int iommu_for_unity_map(struct amd_iommu *iommu,
812 struct unity_map_entry *entry)
816 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
817 bdf = amd_iommu_alias_table[i];
818 if (amd_iommu_rlookup_table[bdf] == iommu)
826 * Init the unity mappings for a specific IOMMU in the system
828 * Basically iterates over all unity mapping entries and applies them to
829 * the default domain DMA of that IOMMU if necessary.
831 static int iommu_init_unity_mappings(struct amd_iommu *iommu)
833 struct unity_map_entry *entry;
836 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
837 if (!iommu_for_unity_map(iommu, entry))
839 ret = dma_ops_unity_map(iommu->default_dom, entry);
848 * This function actually applies the mapping to the page table of the
851 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
852 struct unity_map_entry *e)
857 for (addr = e->address_start; addr < e->address_end;
859 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
864 * if unity mapping is in aperture range mark the page
865 * as allocated in the aperture
867 if (addr < dma_dom->aperture_size)
868 __set_bit(addr >> PAGE_SHIFT,
869 dma_dom->aperture[0]->bitmap);
876 * Inits the unity mappings required for a specific device
878 static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
881 struct unity_map_entry *e;
884 list_for_each_entry(e, &amd_iommu_unity_map, list) {
885 if (!(devid >= e->devid_start && devid <= e->devid_end))
887 ret = dma_ops_unity_map(dma_dom, e);
895 /****************************************************************************
897 * The next functions belong to the address allocator for the dma_ops
898 * interface functions. They work like the allocators in the other IOMMU
899 * drivers. Its basically a bitmap which marks the allocated pages in
900 * the aperture. Maybe it could be enhanced in the future to a more
901 * efficient allocator.
903 ****************************************************************************/
906 * The address allocator core functions.
908 * called with domain->lock held
912 * This function is used to add a new aperture range to an existing
913 * aperture in case of dma_ops domain allocation or address allocation
916 static int alloc_new_range(struct dma_ops_domain *dma_dom,
917 bool populate, gfp_t gfp)
919 int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
920 struct amd_iommu *iommu;
923 #ifdef CONFIG_IOMMU_STRESS
927 if (index >= APERTURE_MAX_RANGES)
930 dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
931 if (!dma_dom->aperture[index])
934 dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
935 if (!dma_dom->aperture[index]->bitmap)
938 dma_dom->aperture[index]->offset = dma_dom->aperture_size;
941 unsigned long address = dma_dom->aperture_size;
942 int i, num_ptes = APERTURE_RANGE_PAGES / 512;
945 for (i = 0; i < num_ptes; ++i) {
946 pte = alloc_pte(&dma_dom->domain, address, PM_MAP_4k,
951 dma_dom->aperture[index]->pte_pages[i] = pte_page;
953 address += APERTURE_RANGE_SIZE / 64;
957 dma_dom->aperture_size += APERTURE_RANGE_SIZE;
959 /* Intialize the exclusion range if necessary */
960 for_each_iommu(iommu) {
961 if (iommu->exclusion_start &&
962 iommu->exclusion_start >= dma_dom->aperture[index]->offset
963 && iommu->exclusion_start < dma_dom->aperture_size) {
964 unsigned long startpage;
965 int pages = iommu_num_pages(iommu->exclusion_start,
966 iommu->exclusion_length,
968 startpage = iommu->exclusion_start >> PAGE_SHIFT;
969 dma_ops_reserve_addresses(dma_dom, startpage, pages);
974 * Check for areas already mapped as present in the new aperture
975 * range and mark those pages as reserved in the allocator. Such
976 * mappings may already exist as a result of requested unity
977 * mappings for devices.
979 for (i = dma_dom->aperture[index]->offset;
980 i < dma_dom->aperture_size;
982 u64 *pte = fetch_pte(&dma_dom->domain, i, PM_MAP_4k);
983 if (!pte || !IOMMU_PTE_PRESENT(*pte))
986 dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1);
989 update_domain(&dma_dom->domain);
994 update_domain(&dma_dom->domain);
996 free_page((unsigned long)dma_dom->aperture[index]->bitmap);
998 kfree(dma_dom->aperture[index]);
999 dma_dom->aperture[index] = NULL;
1004 static unsigned long dma_ops_area_alloc(struct device *dev,
1005 struct dma_ops_domain *dom,
1007 unsigned long align_mask,
1009 unsigned long start)
1011 unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1012 int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1013 int i = start >> APERTURE_RANGE_SHIFT;
1014 unsigned long boundary_size;
1015 unsigned long address = -1;
1016 unsigned long limit;
1018 next_bit >>= PAGE_SHIFT;
1020 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1021 PAGE_SIZE) >> PAGE_SHIFT;
1023 for (;i < max_index; ++i) {
1024 unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
1026 if (dom->aperture[i]->offset >= dma_mask)
1029 limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1030 dma_mask >> PAGE_SHIFT);
1032 address = iommu_area_alloc(dom->aperture[i]->bitmap,
1033 limit, next_bit, pages, 0,
1034 boundary_size, align_mask);
1035 if (address != -1) {
1036 address = dom->aperture[i]->offset +
1037 (address << PAGE_SHIFT);
1038 dom->next_address = address + (pages << PAGE_SHIFT);
1048 static unsigned long dma_ops_alloc_addresses(struct device *dev,
1049 struct dma_ops_domain *dom,
1051 unsigned long align_mask,
1054 unsigned long address;
1056 #ifdef CONFIG_IOMMU_STRESS
1057 dom->next_address = 0;
1058 dom->need_flush = true;
1061 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1062 dma_mask, dom->next_address);
1064 if (address == -1) {
1065 dom->next_address = 0;
1066 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1068 dom->need_flush = true;
1071 if (unlikely(address == -1))
1072 address = DMA_ERROR_CODE;
1074 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1080 * The address free function.
1082 * called with domain->lock held
1084 static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1085 unsigned long address,
1088 unsigned i = address >> APERTURE_RANGE_SHIFT;
1089 struct aperture_range *range = dom->aperture[i];
1091 BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
1093 #ifdef CONFIG_IOMMU_STRESS
1098 if (address >= dom->next_address)
1099 dom->need_flush = true;
1101 address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1103 iommu_area_free(range->bitmap, address, pages);
1107 /****************************************************************************
1109 * The next functions belong to the domain allocation. A domain is
1110 * allocated for every IOMMU as the default domain. If device isolation
1111 * is enabled, every device get its own domain. The most important thing
1112 * about domains is the page table mapping the DMA address space they
1115 ****************************************************************************/
1118 * This function adds a protection domain to the global protection domain list
1120 static void add_domain_to_list(struct protection_domain *domain)
1122 unsigned long flags;
1124 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1125 list_add(&domain->list, &amd_iommu_pd_list);
1126 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1130 * This function removes a protection domain to the global
1131 * protection domain list
1133 static void del_domain_from_list(struct protection_domain *domain)
1135 unsigned long flags;
1137 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1138 list_del(&domain->list);
1139 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1142 static u16 domain_id_alloc(void)
1144 unsigned long flags;
1147 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1148 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1150 if (id > 0 && id < MAX_DOMAIN_ID)
1151 __set_bit(id, amd_iommu_pd_alloc_bitmap);
1154 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1159 static void domain_id_free(int id)
1161 unsigned long flags;
1163 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1164 if (id > 0 && id < MAX_DOMAIN_ID)
1165 __clear_bit(id, amd_iommu_pd_alloc_bitmap);
1166 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1170 * Used to reserve address ranges in the aperture (e.g. for exclusion
1173 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
1174 unsigned long start_page,
1177 unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
1179 if (start_page + pages > last_page)
1180 pages = last_page - start_page;
1182 for (i = start_page; i < start_page + pages; ++i) {
1183 int index = i / APERTURE_RANGE_PAGES;
1184 int page = i % APERTURE_RANGE_PAGES;
1185 __set_bit(page, dom->aperture[index]->bitmap);
1189 static void free_pagetable(struct protection_domain *domain)
1194 p1 = domain->pt_root;
1199 for (i = 0; i < 512; ++i) {
1200 if (!IOMMU_PTE_PRESENT(p1[i]))
1203 p2 = IOMMU_PTE_PAGE(p1[i]);
1204 for (j = 0; j < 512; ++j) {
1205 if (!IOMMU_PTE_PRESENT(p2[j]))
1207 p3 = IOMMU_PTE_PAGE(p2[j]);
1208 free_page((unsigned long)p3);
1211 free_page((unsigned long)p2);
1214 free_page((unsigned long)p1);
1216 domain->pt_root = NULL;
1220 * Free a domain, only used if something went wrong in the
1221 * allocation path and we need to free an already allocated page table
1223 static void dma_ops_domain_free(struct dma_ops_domain *dom)
1230 del_domain_from_list(&dom->domain);
1232 free_pagetable(&dom->domain);
1234 for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
1235 if (!dom->aperture[i])
1237 free_page((unsigned long)dom->aperture[i]->bitmap);
1238 kfree(dom->aperture[i]);
1245 * Allocates a new protection domain usable for the dma_ops functions.
1246 * It also intializes the page table and the address allocator data
1247 * structures required for the dma_ops interface
1249 static struct dma_ops_domain *dma_ops_domain_alloc(void)
1251 struct dma_ops_domain *dma_dom;
1253 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
1257 spin_lock_init(&dma_dom->domain.lock);
1259 dma_dom->domain.id = domain_id_alloc();
1260 if (dma_dom->domain.id == 0)
1262 dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
1263 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1264 dma_dom->domain.flags = PD_DMA_OPS_MASK;
1265 dma_dom->domain.priv = dma_dom;
1266 if (!dma_dom->domain.pt_root)
1269 dma_dom->need_flush = false;
1270 dma_dom->target_dev = 0xffff;
1272 add_domain_to_list(&dma_dom->domain);
1274 if (alloc_new_range(dma_dom, true, GFP_KERNEL))
1278 * mark the first page as allocated so we never return 0 as
1279 * a valid dma-address. So we can use 0 as error value
1281 dma_dom->aperture[0]->bitmap[0] = 1;
1282 dma_dom->next_address = 0;
1288 dma_ops_domain_free(dma_dom);
1294 * little helper function to check whether a given protection domain is a
1297 static bool dma_ops_domain(struct protection_domain *domain)
1299 return domain->flags & PD_DMA_OPS_MASK;
1302 static void set_dte_entry(u16 devid, struct protection_domain *domain)
1304 struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
1305 u64 pte_root = virt_to_phys(domain->pt_root);
1307 BUG_ON(amd_iommu_pd_table[devid] != NULL);
1309 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
1310 << DEV_ENTRY_MODE_SHIFT;
1311 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1313 amd_iommu_dev_table[devid].data[2] = domain->id;
1314 amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
1315 amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
1317 amd_iommu_pd_table[devid] = domain;
1319 /* Do reference counting */
1320 domain->dev_iommu[iommu->index] += 1;
1321 domain->dev_cnt += 1;
1323 /* Flush the changes DTE entry */
1324 iommu_queue_inv_dev_entry(iommu, devid);
1327 static void clear_dte_entry(u16 devid)
1329 struct protection_domain *domain = amd_iommu_pd_table[devid];
1330 struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
1332 BUG_ON(domain == NULL);
1334 /* remove domain from the lookup table */
1335 amd_iommu_pd_table[devid] = NULL;
1337 /* remove entry from the device table seen by the hardware */
1338 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
1339 amd_iommu_dev_table[devid].data[1] = 0;
1340 amd_iommu_dev_table[devid].data[2] = 0;
1342 amd_iommu_apply_erratum_63(devid);
1344 /* decrease reference counters */
1345 domain->dev_iommu[iommu->index] -= 1;
1346 domain->dev_cnt -= 1;
1348 iommu_queue_inv_dev_entry(iommu, devid);
1352 * If a device is not yet associated with a domain, this function does
1353 * assigns it visible for the hardware
1355 static int __attach_device(struct device *dev,
1356 struct protection_domain *domain)
1358 u16 devid = get_device_id(dev);
1359 u16 alias = amd_iommu_alias_table[devid];
1362 spin_lock(&domain->lock);
1364 /* Some sanity checks */
1365 if (amd_iommu_pd_table[alias] != NULL &&
1366 amd_iommu_pd_table[alias] != domain)
1369 if (amd_iommu_pd_table[devid] != NULL &&
1370 amd_iommu_pd_table[devid] != domain)
1373 /* Do real assignment */
1374 if (alias != devid &&
1375 amd_iommu_pd_table[alias] == NULL)
1376 set_dte_entry(alias, domain);
1378 if (amd_iommu_pd_table[devid] == NULL)
1379 set_dte_entry(devid, domain);
1382 spin_unlock(&domain->lock);
1388 * If a device is not yet associated with a domain, this function does
1389 * assigns it visible for the hardware
1391 static int attach_device(struct device *dev,
1392 struct protection_domain *domain)
1394 unsigned long flags;
1397 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1398 ret = __attach_device(dev, domain);
1399 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1402 * We might boot into a crash-kernel here. The crashed kernel
1403 * left the caches in the IOMMU dirty. So we have to flush
1404 * here to evict all dirty stuff.
1406 iommu_flush_tlb_pde(domain);
1412 * Removes a device from a protection domain (unlocked)
1414 static void __detach_device(struct device *dev)
1416 u16 devid = get_device_id(dev);
1417 struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
1421 clear_dte_entry(devid);
1424 * If we run in passthrough mode the device must be assigned to the
1425 * passthrough domain if it is detached from any other domain
1427 if (iommu_pass_through)
1428 __attach_device(dev, pt_domain);
1432 * Removes a device from a protection domain (with devtable_lock held)
1434 static void detach_device(struct device *dev)
1436 unsigned long flags;
1438 /* lock device table */
1439 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1440 __detach_device(dev);
1441 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1445 * Find out the protection domain structure for a given PCI device. This
1446 * will give us the pointer to the page table root for example.
1448 static struct protection_domain *domain_for_device(struct device *dev)
1450 struct protection_domain *dom;
1451 unsigned long flags;
1454 devid = get_device_id(dev);
1455 alias = amd_iommu_alias_table[devid];
1457 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
1458 dom = amd_iommu_pd_table[devid];
1460 amd_iommu_pd_table[alias] != NULL) {
1461 __attach_device(dev, amd_iommu_pd_table[alias]);
1462 dom = amd_iommu_pd_table[devid];
1465 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1470 static int device_change_notifier(struct notifier_block *nb,
1471 unsigned long action, void *data)
1473 struct device *dev = data;
1475 struct protection_domain *domain;
1476 struct dma_ops_domain *dma_domain;
1477 struct amd_iommu *iommu;
1478 unsigned long flags;
1480 if (!check_device(dev))
1483 devid = get_device_id(dev);
1484 iommu = amd_iommu_rlookup_table[devid];
1485 domain = domain_for_device(dev);
1487 if (domain && !dma_ops_domain(domain))
1488 WARN_ONCE(1, "AMD IOMMU WARNING: device %s already bound "
1489 "to a non-dma-ops domain\n", dev_name(dev));
1492 case BUS_NOTIFY_UNBOUND_DRIVER:
1495 if (iommu_pass_through)
1499 case BUS_NOTIFY_ADD_DEVICE:
1500 /* allocate a protection domain if a device is added */
1501 dma_domain = find_protection_domain(devid);
1504 dma_domain = dma_ops_domain_alloc();
1507 dma_domain->target_dev = devid;
1509 spin_lock_irqsave(&iommu_pd_list_lock, flags);
1510 list_add_tail(&dma_domain->list, &iommu_pd_list);
1511 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
1518 iommu_queue_inv_dev_entry(iommu, devid);
1519 iommu_completion_wait(iommu);
1525 static struct notifier_block device_nb = {
1526 .notifier_call = device_change_notifier,
1529 /*****************************************************************************
1531 * The next functions belong to the dma_ops mapping/unmapping code.
1533 *****************************************************************************/
1536 * In the dma_ops path we only have the struct device. This function
1537 * finds the corresponding IOMMU, the protection domain and the
1538 * requestor id for a given device.
1539 * If the device is not yet associated with a domain this is also done
1542 static struct protection_domain *get_domain(struct device *dev)
1544 struct protection_domain *domain;
1545 struct dma_ops_domain *dma_dom;
1546 u16 devid = get_device_id(dev);
1548 if (!check_device(dev))
1549 return ERR_PTR(-EINVAL);
1551 domain = domain_for_device(dev);
1552 if (domain != NULL && !dma_ops_domain(domain))
1553 return ERR_PTR(-EBUSY);
1558 /* Device not bount yet - bind it */
1559 dma_dom = find_protection_domain(devid);
1561 dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
1562 attach_device(dev, &dma_dom->domain);
1563 DUMP_printk("Using protection domain %d for device %s\n",
1564 dma_dom->domain.id, dev_name(dev));
1566 return &dma_dom->domain;
1569 static void update_device_table(struct protection_domain *domain)
1571 unsigned long flags;
1574 for (i = 0; i <= amd_iommu_last_bdf; ++i) {
1575 if (amd_iommu_pd_table[i] != domain)
1577 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1578 set_dte_entry(i, domain);
1579 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1583 static void update_domain(struct protection_domain *domain)
1585 if (!domain->updated)
1588 update_device_table(domain);
1589 flush_devices_by_domain(domain);
1590 iommu_flush_tlb_pde(domain);
1592 domain->updated = false;
1596 * This function fetches the PTE for a given address in the aperture
1598 static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
1599 unsigned long address)
1601 struct aperture_range *aperture;
1602 u64 *pte, *pte_page;
1604 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
1608 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
1610 pte = alloc_pte(&dom->domain, address, PM_MAP_4k, &pte_page,
1612 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
1614 pte += PM_LEVEL_INDEX(0, address);
1616 update_domain(&dom->domain);
1622 * This is the generic map function. It maps one 4kb page at paddr to
1623 * the given address in the DMA address space for the domain.
1625 static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
1626 unsigned long address,
1632 WARN_ON(address > dom->aperture_size);
1636 pte = dma_ops_get_pte(dom, address);
1638 return DMA_ERROR_CODE;
1640 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
1642 if (direction == DMA_TO_DEVICE)
1643 __pte |= IOMMU_PTE_IR;
1644 else if (direction == DMA_FROM_DEVICE)
1645 __pte |= IOMMU_PTE_IW;
1646 else if (direction == DMA_BIDIRECTIONAL)
1647 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
1653 return (dma_addr_t)address;
1657 * The generic unmapping function for on page in the DMA address space.
1659 static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
1660 unsigned long address)
1662 struct aperture_range *aperture;
1665 if (address >= dom->aperture_size)
1668 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
1672 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
1676 pte += PM_LEVEL_INDEX(0, address);
1684 * This function contains common code for mapping of a physically
1685 * contiguous memory region into DMA address space. It is used by all
1686 * mapping functions provided with this IOMMU driver.
1687 * Must be called with the domain lock held.
1689 static dma_addr_t __map_single(struct device *dev,
1690 struct dma_ops_domain *dma_dom,
1697 dma_addr_t offset = paddr & ~PAGE_MASK;
1698 dma_addr_t address, start, ret;
1700 unsigned long align_mask = 0;
1703 pages = iommu_num_pages(paddr, size, PAGE_SIZE);
1706 INC_STATS_COUNTER(total_map_requests);
1709 INC_STATS_COUNTER(cross_page);
1712 align_mask = (1UL << get_order(size)) - 1;
1715 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
1717 if (unlikely(address == DMA_ERROR_CODE)) {
1719 * setting next_address here will let the address
1720 * allocator only scan the new allocated range in the
1721 * first run. This is a small optimization.
1723 dma_dom->next_address = dma_dom->aperture_size;
1725 if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
1729 * aperture was sucessfully enlarged by 128 MB, try
1736 for (i = 0; i < pages; ++i) {
1737 ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
1738 if (ret == DMA_ERROR_CODE)
1746 ADD_STATS_COUNTER(alloced_io_mem, size);
1748 if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
1749 iommu_flush_tlb(&dma_dom->domain);
1750 dma_dom->need_flush = false;
1751 } else if (unlikely(amd_iommu_np_cache))
1752 iommu_flush_pages(&dma_dom->domain, address, size);
1759 for (--i; i >= 0; --i) {
1761 dma_ops_domain_unmap(dma_dom, start);
1764 dma_ops_free_addresses(dma_dom, address, pages);
1766 return DMA_ERROR_CODE;
1770 * Does the reverse of the __map_single function. Must be called with
1771 * the domain lock held too
1773 static void __unmap_single(struct dma_ops_domain *dma_dom,
1774 dma_addr_t dma_addr,
1778 dma_addr_t i, start;
1781 if ((dma_addr == DMA_ERROR_CODE) ||
1782 (dma_addr + size > dma_dom->aperture_size))
1785 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
1786 dma_addr &= PAGE_MASK;
1789 for (i = 0; i < pages; ++i) {
1790 dma_ops_domain_unmap(dma_dom, start);
1794 SUB_STATS_COUNTER(alloced_io_mem, size);
1796 dma_ops_free_addresses(dma_dom, dma_addr, pages);
1798 if (amd_iommu_unmap_flush || dma_dom->need_flush) {
1799 iommu_flush_pages(&dma_dom->domain, dma_addr, size);
1800 dma_dom->need_flush = false;
1805 * The exported map_single function for dma_ops.
1807 static dma_addr_t map_page(struct device *dev, struct page *page,
1808 unsigned long offset, size_t size,
1809 enum dma_data_direction dir,
1810 struct dma_attrs *attrs)
1812 unsigned long flags;
1813 struct protection_domain *domain;
1816 phys_addr_t paddr = page_to_phys(page) + offset;
1818 INC_STATS_COUNTER(cnt_map_single);
1820 domain = get_domain(dev);
1821 if (PTR_ERR(domain) == -EINVAL)
1822 return (dma_addr_t)paddr;
1823 else if (IS_ERR(domain))
1824 return DMA_ERROR_CODE;
1826 dma_mask = *dev->dma_mask;
1828 spin_lock_irqsave(&domain->lock, flags);
1830 addr = __map_single(dev, domain->priv, paddr, size, dir, false,
1832 if (addr == DMA_ERROR_CODE)
1835 iommu_flush_complete(domain);
1838 spin_unlock_irqrestore(&domain->lock, flags);
1844 * The exported unmap_single function for dma_ops.
1846 static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
1847 enum dma_data_direction dir, struct dma_attrs *attrs)
1849 unsigned long flags;
1850 struct protection_domain *domain;
1852 INC_STATS_COUNTER(cnt_unmap_single);
1854 domain = get_domain(dev);
1858 spin_lock_irqsave(&domain->lock, flags);
1860 __unmap_single(domain->priv, dma_addr, size, dir);
1862 iommu_flush_complete(domain);
1864 spin_unlock_irqrestore(&domain->lock, flags);
1868 * This is a special map_sg function which is used if we should map a
1869 * device which is not handled by an AMD IOMMU in the system.
1871 static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
1872 int nelems, int dir)
1874 struct scatterlist *s;
1877 for_each_sg(sglist, s, nelems, i) {
1878 s->dma_address = (dma_addr_t)sg_phys(s);
1879 s->dma_length = s->length;
1886 * The exported map_sg function for dma_ops (handles scatter-gather
1889 static int map_sg(struct device *dev, struct scatterlist *sglist,
1890 int nelems, enum dma_data_direction dir,
1891 struct dma_attrs *attrs)
1893 unsigned long flags;
1894 struct protection_domain *domain;
1896 struct scatterlist *s;
1898 int mapped_elems = 0;
1901 INC_STATS_COUNTER(cnt_map_sg);
1903 domain = get_domain(dev);
1904 if (PTR_ERR(domain) == -EINVAL)
1905 return map_sg_no_iommu(dev, sglist, nelems, dir);
1906 else if (IS_ERR(domain))
1909 dma_mask = *dev->dma_mask;
1911 spin_lock_irqsave(&domain->lock, flags);
1913 for_each_sg(sglist, s, nelems, i) {
1916 s->dma_address = __map_single(dev, domain->priv,
1917 paddr, s->length, dir, false,
1920 if (s->dma_address) {
1921 s->dma_length = s->length;
1927 iommu_flush_complete(domain);
1930 spin_unlock_irqrestore(&domain->lock, flags);
1932 return mapped_elems;
1934 for_each_sg(sglist, s, mapped_elems, i) {
1936 __unmap_single(domain->priv, s->dma_address,
1937 s->dma_length, dir);
1938 s->dma_address = s->dma_length = 0;
1947 * The exported map_sg function for dma_ops (handles scatter-gather
1950 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
1951 int nelems, enum dma_data_direction dir,
1952 struct dma_attrs *attrs)
1954 unsigned long flags;
1955 struct protection_domain *domain;
1956 struct scatterlist *s;
1959 INC_STATS_COUNTER(cnt_unmap_sg);
1961 domain = get_domain(dev);
1965 spin_lock_irqsave(&domain->lock, flags);
1967 for_each_sg(sglist, s, nelems, i) {
1968 __unmap_single(domain->priv, s->dma_address,
1969 s->dma_length, dir);
1970 s->dma_address = s->dma_length = 0;
1973 iommu_flush_complete(domain);
1975 spin_unlock_irqrestore(&domain->lock, flags);
1979 * The exported alloc_coherent function for dma_ops.
1981 static void *alloc_coherent(struct device *dev, size_t size,
1982 dma_addr_t *dma_addr, gfp_t flag)
1984 unsigned long flags;
1986 struct protection_domain *domain;
1988 u64 dma_mask = dev->coherent_dma_mask;
1990 INC_STATS_COUNTER(cnt_alloc_coherent);
1992 domain = get_domain(dev);
1993 if (PTR_ERR(domain) == -EINVAL) {
1994 virt_addr = (void *)__get_free_pages(flag, get_order(size));
1995 *dma_addr = __pa(virt_addr);
1997 } else if (IS_ERR(domain))
2000 dma_mask = dev->coherent_dma_mask;
2001 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2004 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2008 paddr = virt_to_phys(virt_addr);
2011 dma_mask = *dev->dma_mask;
2013 spin_lock_irqsave(&domain->lock, flags);
2015 *dma_addr = __map_single(dev, domain->priv, paddr,
2016 size, DMA_BIDIRECTIONAL, true, dma_mask);
2018 if (*dma_addr == DMA_ERROR_CODE) {
2019 spin_unlock_irqrestore(&domain->lock, flags);
2023 iommu_flush_complete(domain);
2025 spin_unlock_irqrestore(&domain->lock, flags);
2031 free_pages((unsigned long)virt_addr, get_order(size));
2037 * The exported free_coherent function for dma_ops.
2039 static void free_coherent(struct device *dev, size_t size,
2040 void *virt_addr, dma_addr_t dma_addr)
2042 unsigned long flags;
2043 struct protection_domain *domain;
2045 INC_STATS_COUNTER(cnt_free_coherent);
2047 domain = get_domain(dev);
2051 spin_lock_irqsave(&domain->lock, flags);
2053 __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2055 iommu_flush_complete(domain);
2057 spin_unlock_irqrestore(&domain->lock, flags);
2060 free_pages((unsigned long)virt_addr, get_order(size));
2064 * This function is called by the DMA layer to find out if we can handle a
2065 * particular device. It is part of the dma_ops.
2067 static int amd_iommu_dma_supported(struct device *dev, u64 mask)
2069 return check_device(dev);
2073 * The function for pre-allocating protection domains.
2075 * If the driver core informs the DMA layer if a driver grabs a device
2076 * we don't need to preallocate the protection domains anymore.
2077 * For now we have to.
2079 static void prealloc_protection_domains(void)
2081 struct pci_dev *dev = NULL;
2082 struct dma_ops_domain *dma_dom;
2085 while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
2087 /* Do we handle this device? */
2088 if (!check_device(&dev->dev))
2091 /* Is there already any domain for it? */
2092 if (domain_for_device(&dev->dev))
2095 devid = get_device_id(&dev->dev);
2097 dma_dom = dma_ops_domain_alloc();
2100 init_unity_mappings_for_device(dma_dom, devid);
2101 dma_dom->target_dev = devid;
2103 attach_device(&dev->dev, &dma_dom->domain);
2105 list_add_tail(&dma_dom->list, &iommu_pd_list);
2109 static struct dma_map_ops amd_iommu_dma_ops = {
2110 .alloc_coherent = alloc_coherent,
2111 .free_coherent = free_coherent,
2112 .map_page = map_page,
2113 .unmap_page = unmap_page,
2115 .unmap_sg = unmap_sg,
2116 .dma_supported = amd_iommu_dma_supported,
2120 * The function which clues the AMD IOMMU driver into dma_ops.
2122 int __init amd_iommu_init_dma_ops(void)
2124 struct amd_iommu *iommu;
2128 * first allocate a default protection domain for every IOMMU we
2129 * found in the system. Devices not assigned to any other
2130 * protection domain will be assigned to the default one.
2132 for_each_iommu(iommu) {
2133 iommu->default_dom = dma_ops_domain_alloc();
2134 if (iommu->default_dom == NULL)
2136 iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
2137 ret = iommu_init_unity_mappings(iommu);
2143 * If device isolation is enabled, pre-allocate the protection
2144 * domains for each device.
2146 if (amd_iommu_isolate)
2147 prealloc_protection_domains();
2151 #ifdef CONFIG_GART_IOMMU
2152 gart_iommu_aperture_disabled = 1;
2153 gart_iommu_aperture = 0;
2156 /* Make the driver finally visible to the drivers */
2157 dma_ops = &amd_iommu_dma_ops;
2159 register_iommu(&amd_iommu_ops);
2161 bus_register_notifier(&pci_bus_type, &device_nb);
2163 amd_iommu_stats_init();
2169 for_each_iommu(iommu) {
2170 if (iommu->default_dom)
2171 dma_ops_domain_free(iommu->default_dom);
2177 /*****************************************************************************
2179 * The following functions belong to the exported interface of AMD IOMMU
2181 * This interface allows access to lower level functions of the IOMMU
2182 * like protection domain handling and assignement of devices to domains
2183 * which is not possible with the dma_ops interface.
2185 *****************************************************************************/
2187 static void cleanup_domain(struct protection_domain *domain)
2189 unsigned long flags;
2192 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2194 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
2195 if (amd_iommu_pd_table[devid] == domain)
2196 clear_dte_entry(devid);
2198 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2201 static void protection_domain_free(struct protection_domain *domain)
2206 del_domain_from_list(domain);
2209 domain_id_free(domain->id);
2214 static struct protection_domain *protection_domain_alloc(void)
2216 struct protection_domain *domain;
2218 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2222 spin_lock_init(&domain->lock);
2223 domain->id = domain_id_alloc();
2227 add_domain_to_list(domain);
2237 static int amd_iommu_domain_init(struct iommu_domain *dom)
2239 struct protection_domain *domain;
2241 domain = protection_domain_alloc();
2245 domain->mode = PAGE_MODE_3_LEVEL;
2246 domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2247 if (!domain->pt_root)
2255 protection_domain_free(domain);
2260 static void amd_iommu_domain_destroy(struct iommu_domain *dom)
2262 struct protection_domain *domain = dom->priv;
2267 if (domain->dev_cnt > 0)
2268 cleanup_domain(domain);
2270 BUG_ON(domain->dev_cnt != 0);
2272 free_pagetable(domain);
2274 domain_id_free(domain->id);
2281 static void amd_iommu_detach_device(struct iommu_domain *dom,
2284 struct amd_iommu *iommu;
2287 if (!check_device(dev))
2290 devid = get_device_id(dev);
2292 if (amd_iommu_pd_table[devid] != NULL)
2295 iommu = amd_iommu_rlookup_table[devid];
2299 iommu_queue_inv_dev_entry(iommu, devid);
2300 iommu_completion_wait(iommu);
2303 static int amd_iommu_attach_device(struct iommu_domain *dom,
2306 struct protection_domain *domain = dom->priv;
2307 struct protection_domain *old_domain;
2308 struct amd_iommu *iommu;
2312 if (!check_device(dev))
2315 devid = get_device_id(dev);
2317 iommu = amd_iommu_rlookup_table[devid];
2321 old_domain = amd_iommu_pd_table[devid];
2325 ret = attach_device(dev, domain);
2327 iommu_completion_wait(iommu);
2332 static int amd_iommu_map_range(struct iommu_domain *dom,
2333 unsigned long iova, phys_addr_t paddr,
2334 size_t size, int iommu_prot)
2336 struct protection_domain *domain = dom->priv;
2337 unsigned long i, npages = iommu_num_pages(paddr, size, PAGE_SIZE);
2341 if (iommu_prot & IOMMU_READ)
2342 prot |= IOMMU_PROT_IR;
2343 if (iommu_prot & IOMMU_WRITE)
2344 prot |= IOMMU_PROT_IW;
2349 for (i = 0; i < npages; ++i) {
2350 ret = iommu_map_page(domain, iova, paddr, prot, PM_MAP_4k);
2361 static void amd_iommu_unmap_range(struct iommu_domain *dom,
2362 unsigned long iova, size_t size)
2365 struct protection_domain *domain = dom->priv;
2366 unsigned long i, npages = iommu_num_pages(iova, size, PAGE_SIZE);
2370 for (i = 0; i < npages; ++i) {
2371 iommu_unmap_page(domain, iova, PM_MAP_4k);
2375 iommu_flush_tlb_pde(domain);
2378 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2381 struct protection_domain *domain = dom->priv;
2382 unsigned long offset = iova & ~PAGE_MASK;
2386 pte = fetch_pte(domain, iova, PM_MAP_4k);
2388 if (!pte || !IOMMU_PTE_PRESENT(*pte))
2391 paddr = *pte & IOMMU_PAGE_MASK;
2397 static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
2403 static struct iommu_ops amd_iommu_ops = {
2404 .domain_init = amd_iommu_domain_init,
2405 .domain_destroy = amd_iommu_domain_destroy,
2406 .attach_dev = amd_iommu_attach_device,
2407 .detach_dev = amd_iommu_detach_device,
2408 .map = amd_iommu_map_range,
2409 .unmap = amd_iommu_unmap_range,
2410 .iova_to_phys = amd_iommu_iova_to_phys,
2411 .domain_has_cap = amd_iommu_domain_has_cap,
2414 /*****************************************************************************
2416 * The next functions do a basic initialization of IOMMU for pass through
2419 * In passthrough mode the IOMMU is initialized and enabled but not used for
2420 * DMA-API translation.
2422 *****************************************************************************/
2424 int __init amd_iommu_init_passthrough(void)
2426 struct amd_iommu *iommu;
2427 struct pci_dev *dev = NULL;
2430 /* allocate passthroug domain */
2431 pt_domain = protection_domain_alloc();
2435 pt_domain->mode |= PAGE_MODE_NONE;
2437 while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
2439 if (!check_device(&dev->dev))
2442 devid = get_device_id(&dev->dev);
2444 iommu = amd_iommu_rlookup_table[devid];
2448 attach_device(&dev->dev, pt_domain);
2451 pr_info("AMD-Vi: Initialized for Passthrough Mode\n");