#include <linux/device.h>
#include <linux/dma-mapping.h>
+#include <asm/memory.h>
#include <asm/cacheflush.h>
-#include <asm/io.h>
#include <asm/tlbflush.h>
+#include <asm/sizes.h>
+
+/* Sanity check size */
+#if (CONSISTENT_DMA_SIZE % SZ_2M)
+#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
+#endif
-#define CONSISTENT_BASE (0xffc00000)
#define CONSISTENT_END (0xffe00000)
+#define CONSISTENT_BASE (CONSISTENT_END - CONSISTENT_DMA_SIZE)
+
#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
+#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
+#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
+
/*
- * This is the page table (2MB) covering uncached, DMA consistent allocations
+ * These are the page tables (2MB each) covering uncached, DMA consistent allocations
*/
-static pte_t *consistent_pte;
+static pte_t *consistent_pte[NUM_CONSISTENT_PTES];
static DEFINE_SPINLOCK(consistent_lock);
/*
unsigned long vm_start;
unsigned long vm_end;
struct page *vm_pages;
+ int vm_active;
};
static struct vm_region consistent_head = {
};
static struct vm_region *
-vm_region_alloc(struct vm_region *head, size_t size, int gfp)
+vm_region_alloc(struct vm_region *head, size_t size, gfp_t gfp)
{
unsigned long addr = head->vm_start, end = head->vm_end - size;
unsigned long flags;
list_add_tail(&new->vm_list, &c->vm_list);
new->vm_start = addr;
new->vm_end = addr + size;
+ new->vm_active = 1;
spin_unlock_irqrestore(&consistent_lock, flags);
return new;
struct vm_region *c;
list_for_each_entry(c, &head->vm_list, vm_list) {
- if (c->vm_start == addr)
+ if (c->vm_active && c->vm_start == addr)
goto out;
}
c = NULL;
#endif
static void *
-__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, int gfp,
+__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
pgprot_t prot)
{
struct page *page;
unsigned long order;
u64 mask = ISA_DMA_THRESHOLD, limit;
- if (!consistent_pte) {
+ if (!consistent_pte[0]) {
printk(KERN_ERR "%s: not initialised\n", __func__);
dump_stack();
return NULL;
* kernel direct-mapped region for device DMA.
*/
{
- unsigned long kaddr = (unsigned long)page_address(page);
- memset(page_address(page), 0, size);
- dmac_flush_range(kaddr, kaddr + size);
+ void *ptr = page_address(page);
+ memset(ptr, 0, size);
+ dmac_flush_range(ptr, ptr + size);
+ outer_flush_range(__pa(ptr), __pa(ptr) + size);
}
/*
c = vm_region_alloc(&consistent_head, size,
gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
if (c) {
- pte_t *pte = consistent_pte + CONSISTENT_OFFSET(c->vm_start);
+ pte_t *pte;
struct page *end = page + (1 << order);
+ int idx = CONSISTENT_PTE_INDEX(c->vm_start);
+ u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
+ pte = consistent_pte[idx] + off;
c->vm_pages = page;
+ split_page(page, order);
+
/*
* Set the "dma handle"
*/
do {
BUG_ON(!pte_none(*pte));
- set_page_count(page, 1);
/*
* x86 does not mark the pages reserved...
*/
SetPageReserved(page);
- set_pte(pte, mk_pte(page, prot));
+ set_pte_ext(pte, mk_pte(page, prot), 0);
page++;
pte++;
+ off++;
+ if (off >= PTRS_PER_PTE) {
+ off = 0;
+ pte = consistent_pte[++idx];
+ }
} while (size -= PAGE_SIZE);
/*
* Free the otherwise unused pages.
*/
while (page < end) {
- set_page_count(page, 1);
__free_page(page);
page++;
}
* virtual and bus address for that space.
*/
void *
-dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, int gfp)
+dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
{
+ void *memory;
+
+ if (dma_alloc_from_coherent(dev, size, handle, &memory))
+ return memory;
+
+ if (arch_is_coherent()) {
+ void *virt;
+
+ virt = kmalloc(size, gfp);
+ if (!virt)
+ return NULL;
+ *handle = virt_to_dma(dev, virt);
+
+ return virt;
+ }
+
return __dma_alloc(dev, size, handle, gfp,
pgprot_noncached(pgprot_kernel));
}
* dma_alloc_coherent above.
*/
void *
-dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, int gfp)
+dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
{
return __dma_alloc(dev, size, handle, gfp,
pgprot_writecombine(pgprot_kernel));
if (off < kern_size &&
user_size <= (kern_size - off)) {
- vma->vm_flags |= VM_RESERVED;
ret = remap_pfn_range(vma, vma->vm_start,
page_to_pfn(c->vm_pages) + off,
user_size << PAGE_SHIFT,
/*
* free a page as defined by the above mapping.
+ * Must not be called with IRQs disabled.
*/
void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
{
struct vm_region *c;
unsigned long flags, addr;
pte_t *ptep;
+ int idx;
+ u32 off;
+
+ WARN_ON(irqs_disabled());
+
+ if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
+ return;
+
+ if (arch_is_coherent()) {
+ kfree(cpu_addr);
+ return;
+ }
size = PAGE_ALIGN(size);
spin_lock_irqsave(&consistent_lock, flags);
-
c = vm_region_find(&consistent_head, (unsigned long)cpu_addr);
if (!c)
goto no_area;
+ c->vm_active = 0;
+ spin_unlock_irqrestore(&consistent_lock, flags);
+
if ((c->vm_end - c->vm_start) != size) {
printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
__func__, c->vm_end - c->vm_start, size);
size = c->vm_end - c->vm_start;
}
- ptep = consistent_pte + CONSISTENT_OFFSET(c->vm_start);
+ idx = CONSISTENT_PTE_INDEX(c->vm_start);
+ off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
+ ptep = consistent_pte[idx] + off;
addr = c->vm_start;
do {
pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
ptep++;
addr += PAGE_SIZE;
+ off++;
+ if (off >= PTRS_PER_PTE) {
+ off = 0;
+ ptep = consistent_pte[++idx];
+ }
if (!pte_none(pte) && pte_present(pte)) {
pfn = pte_pfn(pte);
flush_tlb_kernel_range(c->vm_start, c->vm_end);
+ spin_lock_irqsave(&consistent_lock, flags);
list_del(&c->vm_list);
-
spin_unlock_irqrestore(&consistent_lock, flags);
kfree(c);
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
- int ret = 0;
-
- spin_lock(&init_mm.page_table_lock);
+ int ret = 0, i = 0;
+ u32 base = CONSISTENT_BASE;
do {
- pgd = pgd_offset(&init_mm, CONSISTENT_BASE);
- pmd = pmd_alloc(&init_mm, pgd, CONSISTENT_BASE);
+ pgd = pgd_offset(&init_mm, base);
+ pmd = pmd_alloc(&init_mm, pgd, base);
if (!pmd) {
printk(KERN_ERR "%s: no pmd tables\n", __func__);
ret = -ENOMEM;
}
WARN_ON(!pmd_none(*pmd));
- pte = pte_alloc_kernel(&init_mm, pmd, CONSISTENT_BASE);
+ pte = pte_alloc_kernel(pmd, base);
if (!pte) {
printk(KERN_ERR "%s: no pte tables\n", __func__);
ret = -ENOMEM;
break;
}
- consistent_pte = pte;
- } while (0);
-
- spin_unlock(&init_mm.page_table_lock);
+ consistent_pte[i++] = pte;
+ base += (1 << PGDIR_SHIFT);
+ } while (base < CONSISTENT_END);
return ret;
}
/*
* Make an area consistent for devices.
+ * Note: Drivers should NOT use this function directly, as it will break
+ * platforms with CONFIG_DMABOUNCE.
+ * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
*/
-void consistent_sync(void *vaddr, size_t size, int direction)
+void dma_cache_maint(const void *start, size_t size, int direction)
{
- unsigned long start = (unsigned long)vaddr;
- unsigned long end = start + size;
+ const void *end = start + size;
+
+ BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(end - 1));
switch (direction) {
case DMA_FROM_DEVICE: /* invalidate only */
dmac_inv_range(start, end);
+ outer_inv_range(__pa(start), __pa(end));
break;
case DMA_TO_DEVICE: /* writeback only */
dmac_clean_range(start, end);
+ outer_clean_range(__pa(start), __pa(end));
break;
case DMA_BIDIRECTIONAL: /* writeback and invalidate */
dmac_flush_range(start, end);
+ outer_flush_range(__pa(start), __pa(end));
break;
default:
BUG();
}
}
-EXPORT_SYMBOL(consistent_sync);
+EXPORT_SYMBOL(dma_cache_maint);
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff