#include <linux/string.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
+#include <linux/crash_dump.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/rtas.h>
#include "plpar_wrappers.h"
-#define DBG(fmt...)
-static void tce_build_pSeries(struct iommu_table *tbl, long index,
+static int tce_build_pSeries(struct iommu_table *tbl, long index,
long npages, unsigned long uaddr,
- enum dma_data_direction direction)
+ enum dma_data_direction direction,
+ struct dma_attrs *attrs)
{
u64 proto_tce;
u64 *tcep;
u64 rpn;
- index <<= TCE_PAGE_FACTOR;
- npages <<= TCE_PAGE_FACTOR;
-
proto_tce = TCE_PCI_READ; // Read allowed
if (direction != DMA_TO_DEVICE)
uaddr += TCE_PAGE_SIZE;
tcep++;
}
+ return 0;
}
{
u64 *tcep;
- npages <<= TCE_PAGE_FACTOR;
- index <<= TCE_PAGE_FACTOR;
-
tcep = ((u64 *)tbl->it_base) + index;
while (npages--)
{
u64 *tcep;
- index <<= TCE_PAGE_FACTOR;
tcep = ((u64 *)tbl->it_base) + index;
return *tcep;
}
-static void tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
+static void tce_free_pSeriesLP(struct iommu_table*, long, long);
+static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
+
+static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
long npages, unsigned long uaddr,
- enum dma_data_direction direction)
+ enum dma_data_direction direction,
+ struct dma_attrs *attrs)
{
- u64 rc;
+ u64 rc = 0;
u64 proto_tce, tce;
u64 rpn;
-
- tcenum <<= TCE_PAGE_FACTOR;
- npages <<= TCE_PAGE_FACTOR;
+ int ret = 0;
+ long tcenum_start = tcenum, npages_start = npages;
rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
proto_tce = TCE_PCI_READ;
tce = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce);
+ if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
+ ret = (int)rc;
+ tce_free_pSeriesLP(tbl, tcenum_start,
+ (npages_start - (npages + 1)));
+ break;
+ }
+
if (rc && printk_ratelimit()) {
- printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
- printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
- printk("\ttcenum = 0x%lx\n", (u64)tcenum);
- printk("\ttce val = 0x%lx\n", tce );
+ printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
+ printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
+ printk("\ttcenum = 0x%llx\n", (u64)tcenum);
+ printk("\ttce val = 0x%llx\n", tce );
show_stack(current, (unsigned long *)__get_SP());
}
tcenum++;
rpn++;
}
+ return ret;
}
static DEFINE_PER_CPU(u64 *, tce_page) = NULL;
-static void tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
+static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
long npages, unsigned long uaddr,
- enum dma_data_direction direction)
+ enum dma_data_direction direction,
+ struct dma_attrs *attrs)
{
- u64 rc;
+ u64 rc = 0;
u64 proto_tce;
u64 *tcep;
u64 rpn;
long l, limit;
+ long tcenum_start = tcenum, npages_start = npages;
+ int ret = 0;
- if (TCE_PAGE_FACTOR == 0 && npages == 1)
+ if (npages == 1) {
return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
- direction);
+ direction, attrs);
+ }
tcep = __get_cpu_var(tce_page);
if (!tcep) {
tcep = (u64 *)__get_free_page(GFP_ATOMIC);
/* If allocation fails, fall back to the loop implementation */
- if (!tcep)
- return tce_build_pSeriesLP(tbl, tcenum, npages,
- uaddr, direction);
+ if (!tcep) {
+ return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
+ direction, attrs);
+ }
__get_cpu_var(tce_page) = tcep;
}
- tcenum <<= TCE_PAGE_FACTOR;
- npages <<= TCE_PAGE_FACTOR;
-
rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
proto_tce = TCE_PCI_READ;
if (direction != DMA_TO_DEVICE)
tcenum += limit;
} while (npages > 0 && !rc);
+ if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
+ ret = (int)rc;
+ tce_freemulti_pSeriesLP(tbl, tcenum_start,
+ (npages_start - (npages + limit)));
+ return ret;
+ }
+
if (rc && printk_ratelimit()) {
- printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
- printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
- printk("\tnpages = 0x%lx\n", (u64)npages);
- printk("\ttce[0] val = 0x%lx\n", tcep[0]);
+ printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
+ printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
+ printk("\tnpages = 0x%llx\n", (u64)npages);
+ printk("\ttce[0] val = 0x%llx\n", tcep[0]);
show_stack(current, (unsigned long *)__get_SP());
}
+ return ret;
}
static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
{
u64 rc;
- tcenum <<= TCE_PAGE_FACTOR;
- npages <<= TCE_PAGE_FACTOR;
-
while (npages--) {
rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);
if (rc && printk_ratelimit()) {
- printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
- printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
- printk("\ttcenum = 0x%lx\n", (u64)tcenum);
+ printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
+ printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
+ printk("\ttcenum = 0x%llx\n", (u64)tcenum);
show_stack(current, (unsigned long *)__get_SP());
}
{
u64 rc;
- tcenum <<= TCE_PAGE_FACTOR;
- npages <<= TCE_PAGE_FACTOR;
-
rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);
if (rc && printk_ratelimit()) {
printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
- printk("\trc = %ld\n", rc);
- printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
- printk("\tnpages = 0x%lx\n", (u64)npages);
+ printk("\trc = %lld\n", rc);
+ printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
+ printk("\tnpages = 0x%llx\n", (u64)npages);
show_stack(current, (unsigned long *)__get_SP());
}
}
u64 rc;
unsigned long tce_ret;
- tcenum <<= TCE_PAGE_FACTOR;
rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);
if (rc && printk_ratelimit()) {
- printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%ld\n",
- rc);
- printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
- printk("\ttcenum = 0x%lx\n", (u64)tcenum);
+ printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
+ printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
+ printk("\ttcenum = 0x%llx\n", (u64)tcenum);
show_stack(current, (unsigned long *)__get_SP());
}
return tce_ret;
}
+#ifdef CONFIG_PCI
static void iommu_table_setparms(struct pci_controller *phb,
struct device_node *dn,
struct iommu_table *tbl)
const unsigned long *basep;
const u32 *sizep;
- node = (struct device_node *)phb->arch_data;
+ node = phb->dn;
- basep = get_property(node, "linux,tce-base", NULL);
- sizep = get_property(node, "linux,tce-size", NULL);
+ basep = of_get_property(node, "linux,tce-base", NULL);
+ sizep = of_get_property(node, "linux,tce-size", NULL);
if (basep == NULL || sizep == NULL) {
printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has "
"missing tce entries !\n", dn->full_name);
tbl->it_base = (unsigned long)__va(*basep);
-#ifndef CONFIG_CRASH_DUMP
- memset((void *)tbl->it_base, 0, *sizep);
-#endif
+ if (!is_kdump_kernel())
+ memset((void *)tbl->it_base, 0, *sizep);
tbl->it_busno = phb->bus->number;
/* Units of tce entries */
- tbl->it_offset = phb->dma_window_base_cur >> PAGE_SHIFT;
+ tbl->it_offset = phb->dma_window_base_cur >> IOMMU_PAGE_SHIFT;
/* Test if we are going over 2GB of DMA space */
if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) {
phb->dma_window_base_cur += phb->dma_window_size;
/* Set the tce table size - measured in entries */
- tbl->it_size = phb->dma_window_size >> PAGE_SHIFT;
+ tbl->it_size = phb->dma_window_size >> IOMMU_PAGE_SHIFT;
tbl->it_index = 0;
tbl->it_blocksize = 16;
static void iommu_table_setparms_lpar(struct pci_controller *phb,
struct device_node *dn,
struct iommu_table *tbl,
- const void *dma_window)
+ const void *dma_window,
+ int bussubno)
{
unsigned long offset, size;
- tbl->it_busno = PCI_DN(dn)->bussubno;
+ tbl->it_busno = bussubno;
of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size);
tbl->it_base = 0;
tbl->it_blocksize = 16;
tbl->it_type = TCE_PCI;
- tbl->it_offset = offset >> PAGE_SHIFT;
- tbl->it_size = size >> PAGE_SHIFT;
+ tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
+ tbl->it_size = size >> IOMMU_PAGE_SHIFT;
}
-static void iommu_bus_setup_pSeries(struct pci_bus *bus)
+static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
{
struct device_node *dn;
struct iommu_table *tbl;
struct pci_dn *pci;
int children;
- DBG("iommu_bus_setup_pSeries, bus %p, bus->self %p\n", bus, bus->self);
-
dn = pci_bus_to_OF_node(bus);
- pci = PCI_DN(dn);
+
+ pr_debug("pci_dma_bus_setup_pSeries: setting up bus %s\n", dn->full_name);
if (bus->self) {
/* This is not a root bus, any setup will be done for the
*/
return;
}
+ pci = PCI_DN(dn);
/* Check if the ISA bus on the system is under
* this PHB.
for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
children++;
- DBG("Children: %d\n", children);
+ pr_debug("Children: %d\n", children);
/* Calculate amount of DMA window per slot. Each window must be
* a power of two (due to pci_alloc_consistent requirements).
while (pci->phb->dma_window_size * children > 0x80000000ul)
pci->phb->dma_window_size >>= 1;
- DBG("No ISA/IDE, window size is 0x%lx\n",
- pci->phb->dma_window_size);
+ pr_debug("No ISA/IDE, window size is 0x%llx\n",
+ pci->phb->dma_window_size);
pci->phb->dma_window_base_cur = 0;
return;
while (pci->phb->dma_window_size * children > 0x70000000ul)
pci->phb->dma_window_size >>= 1;
- DBG("ISA/IDE, window size is 0x%lx\n", pci->phb->dma_window_size);
-
+ pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
}
-static void iommu_bus_setup_pSeriesLP(struct pci_bus *bus)
+static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
{
struct iommu_table *tbl;
struct device_node *dn, *pdn;
struct pci_dn *ppci;
const void *dma_window = NULL;
- DBG("iommu_bus_setup_pSeriesLP, bus %p, bus->self %p\n", bus, bus->self);
-
dn = pci_bus_to_OF_node(bus);
+ pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %s\n",
+ dn->full_name);
+
/* Find nearest ibm,dma-window, walking up the device tree */
for (pdn = dn; pdn != NULL; pdn = pdn->parent) {
- dma_window = get_property(pdn, "ibm,dma-window", NULL);
+ dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
if (dma_window != NULL)
break;
}
if (dma_window == NULL) {
- DBG("iommu_bus_setup_pSeriesLP: bus %s seems to have no ibm,dma-window property\n", dn->full_name);
+ pr_debug(" no ibm,dma-window property !\n");
return;
}
ppci = PCI_DN(pdn);
- if (!ppci->iommu_table) {
- /* Bussubno hasn't been copied yet.
- * Do it now because iommu_table_setparms_lpar needs it.
- */
- ppci->bussubno = bus->number;
+ pr_debug(" parent is %s, iommu_table: 0x%p\n",
+ pdn->full_name, ppci->iommu_table);
+ if (!ppci->iommu_table) {
tbl = kmalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
ppci->phb->node);
-
- iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window);
-
+ iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window,
+ bus->number);
ppci->iommu_table = iommu_init_table(tbl, ppci->phb->node);
+ pr_debug(" created table: %p\n", ppci->iommu_table);
}
if (pdn != dn)
}
-static void iommu_dev_setup_pSeries(struct pci_dev *dev)
+static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
{
- struct device_node *dn, *mydn;
+ struct device_node *dn;
struct iommu_table *tbl;
- DBG("iommu_dev_setup_pSeries, dev %p (%s)\n", dev, pci_name(dev));
+ pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
- mydn = dn = pci_device_to_OF_node(dev);
+ dn = dev->dev.archdata.of_node;
/* If we're the direct child of a root bus, then we need to allocate
* an iommu table ourselves. The bus setup code should have setup
* the window sizes already.
*/
if (!dev->bus->self) {
- DBG(" --> first child, no bridge. Allocating iommu table.\n");
- tbl = kmalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
- PCI_DN(dn)->phb->node);
- iommu_table_setparms(PCI_DN(dn)->phb, dn, tbl);
- PCI_DN(dn)->iommu_table = iommu_init_table(tbl,
- PCI_DN(dn)->phb->node);
+ struct pci_controller *phb = PCI_DN(dn)->phb;
+ pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
+ tbl = kmalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
+ phb->node);
+ iommu_table_setparms(phb, dn, tbl);
+ PCI_DN(dn)->iommu_table = iommu_init_table(tbl, phb->node);
+ set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
return;
}
while (dn && PCI_DN(dn) && PCI_DN(dn)->iommu_table == NULL)
dn = dn->parent;
- if (dn && PCI_DN(dn)) {
- PCI_DN(mydn)->iommu_table = PCI_DN(dn)->iommu_table;
- } else {
- DBG("iommu_dev_setup_pSeries, dev %p (%s) has no iommu table\n", dev, pci_name(dev));
- }
-}
-
-static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node)
-{
- int err = NOTIFY_OK;
- struct device_node *np = node;
- struct pci_dn *pci = PCI_DN(np);
-
- switch (action) {
- case PSERIES_RECONFIG_REMOVE:
- if (pci && pci->iommu_table &&
- get_property(np, "ibm,dma-window", NULL))
- iommu_free_table(np);
- break;
- default:
- err = NOTIFY_DONE;
- break;
- }
- return err;
+ if (dn && PCI_DN(dn))
+ set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
+ else
+ printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
+ pci_name(dev));
}
-static struct notifier_block iommu_reconfig_nb = {
- .notifier_call = iommu_reconfig_notifier,
-};
-
-static void iommu_dev_setup_pSeriesLP(struct pci_dev *dev)
+static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
{
struct device_node *pdn, *dn;
struct iommu_table *tbl;
const void *dma_window = NULL;
struct pci_dn *pci;
- DBG("iommu_dev_setup_pSeriesLP, dev %p (%s)\n", dev, pci_name(dev));
+ pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
/* dev setup for LPAR is a little tricky, since the device tree might
* contain the dma-window properties per-device and not neccesarily
* already allocated.
*/
dn = pci_device_to_OF_node(dev);
+ pr_debug(" node is %s\n", dn->full_name);
for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
pdn = pdn->parent) {
- dma_window = get_property(pdn, "ibm,dma-window", NULL);
+ dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
if (dma_window)
break;
}
+ if (!pdn || !PCI_DN(pdn)) {
+ printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
+ "no DMA window found for pci dev=%s dn=%s\n",
+ pci_name(dev), dn? dn->full_name : "<null>");
+ return;
+ }
+ pr_debug(" parent is %s\n", pdn->full_name);
+
/* Check for parent == NULL so we don't try to setup the empty EADS
* slots on POWER4 machines.
*/
if (dma_window == NULL || pdn->parent == NULL) {
- DBG("No dma window for device, linking to parent\n");
- PCI_DN(dn)->iommu_table = PCI_DN(pdn)->iommu_table;
+ pr_debug(" no dma window for device, linking to parent\n");
+ set_iommu_table_base(&dev->dev, PCI_DN(pdn)->iommu_table);
return;
- } else {
- DBG("Found DMA window, allocating table\n");
}
pci = PCI_DN(pdn);
if (!pci->iommu_table) {
- /* iommu_table_setparms_lpar needs bussubno. */
- pci->bussubno = pci->phb->bus->number;
-
tbl = kmalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
pci->phb->node);
-
- iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window);
-
+ iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window,
+ pci->phb->bus->number);
pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
+ pr_debug(" created table: %p\n", pci->iommu_table);
+ } else {
+ pr_debug(" found DMA window, table: %p\n", pci->iommu_table);
}
- if (pdn != dn)
- PCI_DN(dn)->iommu_table = pci->iommu_table;
+ set_iommu_table_base(&dev->dev, pci->iommu_table);
}
+#else /* CONFIG_PCI */
+#define pci_dma_bus_setup_pSeries NULL
+#define pci_dma_dev_setup_pSeries NULL
+#define pci_dma_bus_setup_pSeriesLP NULL
+#define pci_dma_dev_setup_pSeriesLP NULL
+#endif /* !CONFIG_PCI */
-static void iommu_bus_setup_null(struct pci_bus *b) { }
-static void iommu_dev_setup_null(struct pci_dev *d) { }
+static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node)
+{
+ int err = NOTIFY_OK;
+ struct device_node *np = node;
+ struct pci_dn *pci = PCI_DN(np);
+
+ switch (action) {
+ case PSERIES_RECONFIG_REMOVE:
+ if (pci && pci->iommu_table &&
+ of_get_property(np, "ibm,dma-window", NULL))
+ iommu_free_table(pci->iommu_table, np->full_name);
+ break;
+ default:
+ err = NOTIFY_DONE;
+ break;
+ }
+ return err;
+}
+
+static struct notifier_block iommu_reconfig_nb = {
+ .notifier_call = iommu_reconfig_notifier,
+};
/* These are called very early. */
void iommu_init_early_pSeries(void)
{
- if (of_chosen && get_property(of_chosen, "linux,iommu-off", NULL)) {
+ if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL)) {
/* Direct I/O, IOMMU off */
- ppc_md.iommu_dev_setup = iommu_dev_setup_null;
- ppc_md.iommu_bus_setup = iommu_bus_setup_null;
- pci_direct_iommu_init();
-
+ ppc_md.pci_dma_dev_setup = NULL;
+ ppc_md.pci_dma_bus_setup = NULL;
+ set_pci_dma_ops(&dma_direct_ops);
return;
}
ppc_md.tce_free = tce_free_pSeriesLP;
}
ppc_md.tce_get = tce_get_pSeriesLP;
- ppc_md.iommu_bus_setup = iommu_bus_setup_pSeriesLP;
- ppc_md.iommu_dev_setup = iommu_dev_setup_pSeriesLP;
+ ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
+ ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
} else {
ppc_md.tce_build = tce_build_pSeries;
ppc_md.tce_free = tce_free_pSeries;
ppc_md.tce_get = tce_get_pseries;
- ppc_md.iommu_bus_setup = iommu_bus_setup_pSeries;
- ppc_md.iommu_dev_setup = iommu_dev_setup_pSeries;
+ ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeries;
+ ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeries;
}
pSeries_reconfig_notifier_register(&iommu_reconfig_nb);
- pci_iommu_init();
+ set_pci_dma_ops(&dma_iommu_ops);
}
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff