2 * Renesas SuperH DMA Engine support
4 * base is drivers/dma/flsdma.c
6 * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
7 * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
8 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
10 * This is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * - DMA of SuperH does not have Hardware DMA chain mode.
16 * - MAX DMA size is 16MB.
20 #include <linux/init.h>
21 #include <linux/module.h>
22 #include <linux/interrupt.h>
23 #include <linux/dmaengine.h>
24 #include <linux/delay.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/platform_device.h>
27 #include <asm/dmaengine.h>
30 /* DMA descriptor control */
31 enum sh_dmae_desc_status {
35 DESC_COMPLETED, /* completed, have to call callback */
36 DESC_WAITING, /* callback called, waiting for ack / re-submit */
39 #define NR_DESCS_PER_CHANNEL 32
40 /* Default MEMCPY transfer size = 2^2 = 4 bytes */
41 #define LOG2_DEFAULT_XFER_SIZE 2
43 /* A bitmask with bits enough for enum sh_dmae_slave_chan_id */
44 static unsigned long sh_dmae_slave_used[BITS_TO_LONGS(SHDMA_SLAVE_NUMBER)];
46 static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all);
48 static void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg)
50 __raw_writel(data, sh_dc->base + reg / sizeof(u32));
53 static u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg)
55 return __raw_readl(sh_dc->base + reg / sizeof(u32));
58 static u16 dmaor_read(struct sh_dmae_device *shdev)
60 return __raw_readw(shdev->chan_reg + DMAOR / sizeof(u32));
63 static void dmaor_write(struct sh_dmae_device *shdev, u16 data)
65 __raw_writew(data, shdev->chan_reg + DMAOR / sizeof(u32));
69 * Reset DMA controller
71 * SH7780 has two DMAOR register
73 static void sh_dmae_ctl_stop(struct sh_dmae_device *shdev)
75 unsigned short dmaor = dmaor_read(shdev);
77 dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME));
80 static int sh_dmae_rst(struct sh_dmae_device *shdev)
84 sh_dmae_ctl_stop(shdev);
85 dmaor = dmaor_read(shdev) | shdev->pdata->dmaor_init;
87 dmaor_write(shdev, dmaor);
88 if (dmaor_read(shdev) & (DMAOR_AE | DMAOR_NMIF)) {
89 pr_warning("dma-sh: Can't initialize DMAOR.\n");
95 static bool dmae_is_busy(struct sh_dmae_chan *sh_chan)
97 u32 chcr = sh_dmae_readl(sh_chan, CHCR);
99 if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE)
100 return true; /* working */
102 return false; /* waiting */
105 static unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr)
107 struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
108 struct sh_dmae_device, common);
109 struct sh_dmae_pdata *pdata = shdev->pdata;
110 int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) |
111 ((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift);
113 if (cnt >= pdata->ts_shift_num)
116 return pdata->ts_shift[cnt];
119 static u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size)
121 struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
122 struct sh_dmae_device, common);
123 struct sh_dmae_pdata *pdata = shdev->pdata;
126 for (i = 0; i < pdata->ts_shift_num; i++)
127 if (pdata->ts_shift[i] == l2size)
130 if (i == pdata->ts_shift_num)
133 return ((i << pdata->ts_low_shift) & pdata->ts_low_mask) |
134 ((i << pdata->ts_high_shift) & pdata->ts_high_mask);
137 static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw)
139 sh_dmae_writel(sh_chan, hw->sar, SAR);
140 sh_dmae_writel(sh_chan, hw->dar, DAR);
141 sh_dmae_writel(sh_chan, hw->tcr >> sh_chan->xmit_shift, TCR);
144 static void dmae_start(struct sh_dmae_chan *sh_chan)
146 u32 chcr = sh_dmae_readl(sh_chan, CHCR);
148 chcr |= CHCR_DE | CHCR_IE;
149 sh_dmae_writel(sh_chan, chcr & ~CHCR_TE, CHCR);
152 static void dmae_halt(struct sh_dmae_chan *sh_chan)
154 u32 chcr = sh_dmae_readl(sh_chan, CHCR);
156 chcr &= ~(CHCR_DE | CHCR_TE | CHCR_IE);
157 sh_dmae_writel(sh_chan, chcr, CHCR);
160 static void dmae_init(struct sh_dmae_chan *sh_chan)
163 * Default configuration for dual address memory-memory transfer.
164 * 0x400 represents auto-request.
166 u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan,
167 LOG2_DEFAULT_XFER_SIZE);
168 sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr);
169 sh_dmae_writel(sh_chan, chcr, CHCR);
172 static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
174 /* When DMA was working, can not set data to CHCR */
175 if (dmae_is_busy(sh_chan))
178 sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val);
179 sh_dmae_writel(sh_chan, val, CHCR);
184 static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
186 struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
187 struct sh_dmae_device, common);
188 struct sh_dmae_pdata *pdata = shdev->pdata;
189 struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id];
190 u16 __iomem *addr = shdev->dmars + chan_pdata->dmars / sizeof(u16);
191 int shift = chan_pdata->dmars_bit;
193 if (dmae_is_busy(sh_chan))
196 __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift),
202 static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx)
204 struct sh_desc *desc = tx_to_sh_desc(tx), *chunk, *last = desc, *c;
205 struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan);
206 dma_async_tx_callback callback = tx->callback;
209 spin_lock_bh(&sh_chan->desc_lock);
211 cookie = sh_chan->common.cookie;
216 sh_chan->common.cookie = cookie;
219 /* Mark all chunks of this descriptor as submitted, move to the queue */
220 list_for_each_entry_safe(chunk, c, desc->node.prev, node) {
222 * All chunks are on the global ld_free, so, we have to find
223 * the end of the chain ourselves
225 if (chunk != desc && (chunk->mark == DESC_IDLE ||
226 chunk->async_tx.cookie > 0 ||
227 chunk->async_tx.cookie == -EBUSY ||
228 &chunk->node == &sh_chan->ld_free))
230 chunk->mark = DESC_SUBMITTED;
231 /* Callback goes to the last chunk */
232 chunk->async_tx.callback = NULL;
233 chunk->cookie = cookie;
234 list_move_tail(&chunk->node, &sh_chan->ld_queue);
238 last->async_tx.callback = callback;
239 last->async_tx.callback_param = tx->callback_param;
241 dev_dbg(sh_chan->dev, "submit #%d@%p on %d: %x[%d] -> %x\n",
242 tx->cookie, &last->async_tx, sh_chan->id,
243 desc->hw.sar, desc->hw.tcr, desc->hw.dar);
245 spin_unlock_bh(&sh_chan->desc_lock);
250 /* Called with desc_lock held */
251 static struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan)
253 struct sh_desc *desc;
255 list_for_each_entry(desc, &sh_chan->ld_free, node)
256 if (desc->mark != DESC_PREPARED) {
257 BUG_ON(desc->mark != DESC_IDLE);
258 list_del(&desc->node);
265 static struct sh_dmae_slave_config *sh_dmae_find_slave(
266 struct sh_dmae_chan *sh_chan, enum sh_dmae_slave_chan_id slave_id)
268 struct dma_device *dma_dev = sh_chan->common.device;
269 struct sh_dmae_device *shdev = container_of(dma_dev,
270 struct sh_dmae_device, common);
271 struct sh_dmae_pdata *pdata = shdev->pdata;
274 if ((unsigned)slave_id >= SHDMA_SLAVE_NUMBER)
277 for (i = 0; i < pdata->slave_num; i++)
278 if (pdata->slave[i].slave_id == slave_id)
279 return pdata->slave + i;
284 static int sh_dmae_alloc_chan_resources(struct dma_chan *chan)
286 struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
287 struct sh_desc *desc;
288 struct sh_dmae_slave *param = chan->private;
291 * This relies on the guarantee from dmaengine that alloc_chan_resources
292 * never runs concurrently with itself or free_chan_resources.
295 struct sh_dmae_slave_config *cfg;
297 cfg = sh_dmae_find_slave(sh_chan, param->slave_id);
301 if (test_and_set_bit(param->slave_id, sh_dmae_slave_used))
306 dmae_set_dmars(sh_chan, cfg->mid_rid);
307 dmae_set_chcr(sh_chan, cfg->chcr);
308 } else if ((sh_dmae_readl(sh_chan, CHCR) & 0xf00) != 0x400) {
312 spin_lock_bh(&sh_chan->desc_lock);
313 while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) {
314 spin_unlock_bh(&sh_chan->desc_lock);
315 desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL);
317 spin_lock_bh(&sh_chan->desc_lock);
320 dma_async_tx_descriptor_init(&desc->async_tx,
322 desc->async_tx.tx_submit = sh_dmae_tx_submit;
323 desc->mark = DESC_IDLE;
325 spin_lock_bh(&sh_chan->desc_lock);
326 list_add(&desc->node, &sh_chan->ld_free);
327 sh_chan->descs_allocated++;
329 spin_unlock_bh(&sh_chan->desc_lock);
331 return sh_chan->descs_allocated;
335 * sh_dma_free_chan_resources - Free all resources of the channel.
337 static void sh_dmae_free_chan_resources(struct dma_chan *chan)
339 struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
340 struct sh_desc *desc, *_desc;
345 /* Prepared and not submitted descriptors can still be on the queue */
346 if (!list_empty(&sh_chan->ld_queue))
347 sh_dmae_chan_ld_cleanup(sh_chan, true);
350 /* The caller is holding dma_list_mutex */
351 struct sh_dmae_slave *param = chan->private;
352 clear_bit(param->slave_id, sh_dmae_slave_used);
355 spin_lock_bh(&sh_chan->desc_lock);
357 list_splice_init(&sh_chan->ld_free, &list);
358 sh_chan->descs_allocated = 0;
360 spin_unlock_bh(&sh_chan->desc_lock);
362 list_for_each_entry_safe(desc, _desc, &list, node)
367 * sh_dmae_add_desc - get, set up and return one transfer descriptor
368 * @sh_chan: DMA channel
369 * @flags: DMA transfer flags
370 * @dest: destination DMA address, incremented when direction equals
371 * DMA_FROM_DEVICE or DMA_BIDIRECTIONAL
372 * @src: source DMA address, incremented when direction equals
373 * DMA_TO_DEVICE or DMA_BIDIRECTIONAL
374 * @len: DMA transfer length
375 * @first: if NULL, set to the current descriptor and cookie set to -EBUSY
376 * @direction: needed for slave DMA to decide which address to keep constant,
377 * equals DMA_BIDIRECTIONAL for MEMCPY
378 * Returns 0 or an error
379 * Locks: called with desc_lock held
381 static struct sh_desc *sh_dmae_add_desc(struct sh_dmae_chan *sh_chan,
382 unsigned long flags, dma_addr_t *dest, dma_addr_t *src, size_t *len,
383 struct sh_desc **first, enum dma_data_direction direction)
391 /* Allocate the link descriptor from the free list */
392 new = sh_dmae_get_desc(sh_chan);
394 dev_err(sh_chan->dev, "No free link descriptor available\n");
398 copy_size = min(*len, (size_t)SH_DMA_TCR_MAX + 1);
402 new->hw.tcr = copy_size;
406 new->async_tx.cookie = -EBUSY;
409 /* Other desc - invisible to the user */
410 new->async_tx.cookie = -EINVAL;
413 dev_dbg(sh_chan->dev,
414 "chaining (%u/%u)@%x -> %x with %p, cookie %d, shift %d\n",
415 copy_size, *len, *src, *dest, &new->async_tx,
416 new->async_tx.cookie, sh_chan->xmit_shift);
418 new->mark = DESC_PREPARED;
419 new->async_tx.flags = flags;
420 new->direction = direction;
423 if (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE)
425 if (direction == DMA_BIDIRECTIONAL || direction == DMA_FROM_DEVICE)
432 * sh_dmae_prep_sg - prepare transfer descriptors from an SG list
434 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also
435 * converted to scatter-gather to guarantee consistent locking and a correct
436 * list manipulation. For slave DMA direction carries the usual meaning, and,
437 * logically, the SG list is RAM and the addr variable contains slave address,
438 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_BIDIRECTIONAL
439 * and the SG list contains only one element and points at the source buffer.
441 static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_chan,
442 struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr,
443 enum dma_data_direction direction, unsigned long flags)
445 struct scatterlist *sg;
446 struct sh_desc *first = NULL, *new = NULL /* compiler... */;
454 for_each_sg(sgl, sg, sg_len, i)
455 chunks += (sg_dma_len(sg) + SH_DMA_TCR_MAX) /
456 (SH_DMA_TCR_MAX + 1);
458 /* Have to lock the whole loop to protect against concurrent release */
459 spin_lock_bh(&sh_chan->desc_lock);
463 * first descriptor is what user is dealing with in all API calls, its
464 * cookie is at first set to -EBUSY, at tx-submit to a positive
466 * if more than one chunk is needed further chunks have cookie = -EINVAL
467 * the last chunk, if not equal to the first, has cookie = -ENOSPC
468 * all chunks are linked onto the tx_list head with their .node heads
469 * only during this function, then they are immediately spliced
470 * back onto the free list in form of a chain
472 for_each_sg(sgl, sg, sg_len, i) {
473 dma_addr_t sg_addr = sg_dma_address(sg);
474 size_t len = sg_dma_len(sg);
480 dev_dbg(sh_chan->dev, "Add SG #%d@%p[%d], dma %llx\n",
481 i, sg, len, (unsigned long long)sg_addr);
483 if (direction == DMA_FROM_DEVICE)
484 new = sh_dmae_add_desc(sh_chan, flags,
485 &sg_addr, addr, &len, &first,
488 new = sh_dmae_add_desc(sh_chan, flags,
489 addr, &sg_addr, &len, &first,
494 new->chunks = chunks--;
495 list_add_tail(&new->node, &tx_list);
500 new->async_tx.cookie = -ENOSPC;
502 /* Put them back on the free list, so, they don't get lost */
503 list_splice_tail(&tx_list, &sh_chan->ld_free);
505 spin_unlock_bh(&sh_chan->desc_lock);
507 return &first->async_tx;
510 list_for_each_entry(new, &tx_list, node)
511 new->mark = DESC_IDLE;
512 list_splice(&tx_list, &sh_chan->ld_free);
514 spin_unlock_bh(&sh_chan->desc_lock);
519 static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy(
520 struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src,
521 size_t len, unsigned long flags)
523 struct sh_dmae_chan *sh_chan;
524 struct scatterlist sg;
529 chan->private = NULL;
531 sh_chan = to_sh_chan(chan);
533 sg_init_table(&sg, 1);
534 sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len,
535 offset_in_page(dma_src));
536 sg_dma_address(&sg) = dma_src;
537 sg_dma_len(&sg) = len;
539 return sh_dmae_prep_sg(sh_chan, &sg, 1, &dma_dest, DMA_BIDIRECTIONAL,
543 static struct dma_async_tx_descriptor *sh_dmae_prep_slave_sg(
544 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
545 enum dma_data_direction direction, unsigned long flags)
547 struct sh_dmae_slave *param;
548 struct sh_dmae_chan *sh_chan;
553 sh_chan = to_sh_chan(chan);
554 param = chan->private;
556 /* Someone calling slave DMA on a public channel? */
557 if (!param || !sg_len) {
558 dev_warn(sh_chan->dev, "%s: bad parameter: %p, %d, %d\n",
559 __func__, param, sg_len, param ? param->slave_id : -1);
564 * if (param != NULL), this is a successfully requested slave channel,
565 * therefore param->config != NULL too.
567 return sh_dmae_prep_sg(sh_chan, sgl, sg_len, ¶m->config->addr,
571 static void sh_dmae_terminate_all(struct dma_chan *chan)
573 struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
578 sh_dmae_chan_ld_cleanup(sh_chan, true);
581 static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all)
583 struct sh_desc *desc, *_desc;
584 /* Is the "exposed" head of a chain acked? */
585 bool head_acked = false;
586 dma_cookie_t cookie = 0;
587 dma_async_tx_callback callback = NULL;
590 spin_lock_bh(&sh_chan->desc_lock);
591 list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) {
592 struct dma_async_tx_descriptor *tx = &desc->async_tx;
594 BUG_ON(tx->cookie > 0 && tx->cookie != desc->cookie);
595 BUG_ON(desc->mark != DESC_SUBMITTED &&
596 desc->mark != DESC_COMPLETED &&
597 desc->mark != DESC_WAITING);
600 * queue is ordered, and we use this loop to (1) clean up all
601 * completed descriptors, and to (2) update descriptor flags of
602 * any chunks in a (partially) completed chain
604 if (!all && desc->mark == DESC_SUBMITTED &&
605 desc->cookie != cookie)
611 if (desc->mark == DESC_COMPLETED && desc->chunks == 1) {
612 if (sh_chan->completed_cookie != desc->cookie - 1)
613 dev_dbg(sh_chan->dev,
614 "Completing cookie %d, expected %d\n",
616 sh_chan->completed_cookie + 1);
617 sh_chan->completed_cookie = desc->cookie;
620 /* Call callback on the last chunk */
621 if (desc->mark == DESC_COMPLETED && tx->callback) {
622 desc->mark = DESC_WAITING;
623 callback = tx->callback;
624 param = tx->callback_param;
625 dev_dbg(sh_chan->dev, "descriptor #%d@%p on %d callback\n",
626 tx->cookie, tx, sh_chan->id);
627 BUG_ON(desc->chunks != 1);
631 if (tx->cookie > 0 || tx->cookie == -EBUSY) {
632 if (desc->mark == DESC_COMPLETED) {
633 BUG_ON(tx->cookie < 0);
634 desc->mark = DESC_WAITING;
636 head_acked = async_tx_test_ack(tx);
638 switch (desc->mark) {
640 desc->mark = DESC_WAITING;
644 async_tx_ack(&desc->async_tx);
648 dev_dbg(sh_chan->dev, "descriptor %p #%d completed.\n",
651 if (((desc->mark == DESC_COMPLETED ||
652 desc->mark == DESC_WAITING) &&
653 async_tx_test_ack(&desc->async_tx)) || all) {
654 /* Remove from ld_queue list */
655 desc->mark = DESC_IDLE;
656 list_move(&desc->node, &sh_chan->ld_free);
659 spin_unlock_bh(&sh_chan->desc_lock);
668 * sh_chan_ld_cleanup - Clean up link descriptors
670 * This function cleans up the ld_queue of DMA channel.
672 static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all)
674 while (__ld_cleanup(sh_chan, all))
678 static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
680 struct sh_desc *desc;
682 spin_lock_bh(&sh_chan->desc_lock);
684 if (dmae_is_busy(sh_chan)) {
685 spin_unlock_bh(&sh_chan->desc_lock);
689 /* Find the first not transferred desciptor */
690 list_for_each_entry(desc, &sh_chan->ld_queue, node)
691 if (desc->mark == DESC_SUBMITTED) {
692 /* Get the ld start address from ld_queue */
693 dmae_set_reg(sh_chan, &desc->hw);
698 spin_unlock_bh(&sh_chan->desc_lock);
701 static void sh_dmae_memcpy_issue_pending(struct dma_chan *chan)
703 struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
704 sh_chan_xfer_ld_queue(sh_chan);
707 static enum dma_status sh_dmae_is_complete(struct dma_chan *chan,
712 struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
713 dma_cookie_t last_used;
714 dma_cookie_t last_complete;
715 enum dma_status status;
717 sh_dmae_chan_ld_cleanup(sh_chan, false);
719 last_used = chan->cookie;
720 last_complete = sh_chan->completed_cookie;
721 BUG_ON(last_complete < 0);
724 *done = last_complete;
729 spin_lock_bh(&sh_chan->desc_lock);
731 status = dma_async_is_complete(cookie, last_complete, last_used);
734 * If we don't find cookie on the queue, it has been aborted and we have
737 if (status != DMA_SUCCESS) {
738 struct sh_desc *desc;
740 list_for_each_entry(desc, &sh_chan->ld_queue, node)
741 if (desc->cookie == cookie) {
742 status = DMA_IN_PROGRESS;
747 spin_unlock_bh(&sh_chan->desc_lock);
752 static irqreturn_t sh_dmae_interrupt(int irq, void *data)
754 irqreturn_t ret = IRQ_NONE;
755 struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
756 u32 chcr = sh_dmae_readl(sh_chan, CHCR);
758 if (chcr & CHCR_TE) {
763 tasklet_schedule(&sh_chan->tasklet);
769 #if defined(CONFIG_CPU_SH4)
770 static irqreturn_t sh_dmae_err(int irq, void *data)
772 struct sh_dmae_device *shdev = (struct sh_dmae_device *)data;
775 /* halt the dma controller */
776 sh_dmae_ctl_stop(shdev);
778 /* We cannot detect, which channel caused the error, have to reset all */
779 for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) {
780 struct sh_dmae_chan *sh_chan = shdev->chan[i];
782 struct sh_desc *desc;
783 /* Stop the channel */
786 list_for_each_entry(desc, &sh_chan->ld_queue, node) {
787 struct dma_async_tx_descriptor *tx = &desc->async_tx;
788 desc->mark = DESC_IDLE;
790 tx->callback(tx->callback_param);
792 list_splice_init(&sh_chan->ld_queue, &sh_chan->ld_free);
801 static void dmae_do_tasklet(unsigned long data)
803 struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
804 struct sh_desc *desc;
805 u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
806 u32 dar_buf = sh_dmae_readl(sh_chan, DAR);
808 spin_lock(&sh_chan->desc_lock);
809 list_for_each_entry(desc, &sh_chan->ld_queue, node) {
810 if (desc->mark == DESC_SUBMITTED &&
811 ((desc->direction == DMA_FROM_DEVICE &&
812 (desc->hw.dar + desc->hw.tcr) == dar_buf) ||
813 (desc->hw.sar + desc->hw.tcr) == sar_buf)) {
814 dev_dbg(sh_chan->dev, "done #%d@%p dst %u\n",
815 desc->async_tx.cookie, &desc->async_tx,
817 desc->mark = DESC_COMPLETED;
821 spin_unlock(&sh_chan->desc_lock);
824 sh_chan_xfer_ld_queue(sh_chan);
825 sh_dmae_chan_ld_cleanup(sh_chan, false);
828 static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id,
829 int irq, unsigned long flags)
832 struct sh_dmae_channel *chan_pdata = &shdev->pdata->channel[id];
833 struct platform_device *pdev = to_platform_device(shdev->common.dev);
834 struct sh_dmae_chan *new_sh_chan;
837 new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL);
839 dev_err(shdev->common.dev,
840 "No free memory for allocating dma channels!\n");
844 /* copy struct dma_device */
845 new_sh_chan->common.device = &shdev->common;
847 new_sh_chan->dev = shdev->common.dev;
848 new_sh_chan->id = id;
849 new_sh_chan->irq = irq;
850 new_sh_chan->base = shdev->chan_reg + chan_pdata->offset / sizeof(u32);
852 /* Init DMA tasklet */
853 tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet,
854 (unsigned long)new_sh_chan);
856 /* Init the channel */
857 dmae_init(new_sh_chan);
859 spin_lock_init(&new_sh_chan->desc_lock);
861 /* Init descripter manage list */
862 INIT_LIST_HEAD(&new_sh_chan->ld_queue);
863 INIT_LIST_HEAD(&new_sh_chan->ld_free);
865 /* Add the channel to DMA device channel list */
866 list_add_tail(&new_sh_chan->common.device_node,
867 &shdev->common.channels);
868 shdev->common.chancnt++;
871 snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id),
872 "sh-dmae%d.%d", pdev->id, new_sh_chan->id);
874 snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id),
875 "sh-dma%d", new_sh_chan->id);
877 /* set up channel irq */
878 err = request_irq(irq, &sh_dmae_interrupt, flags,
879 new_sh_chan->dev_id, new_sh_chan);
881 dev_err(shdev->common.dev, "DMA channel %d request_irq error "
882 "with return %d\n", id, err);
886 shdev->chan[id] = new_sh_chan;
890 /* remove from dmaengine device node */
891 list_del(&new_sh_chan->common.device_node);
896 static void sh_dmae_chan_remove(struct sh_dmae_device *shdev)
900 for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) {
901 if (shdev->chan[i]) {
902 struct sh_dmae_chan *sh_chan = shdev->chan[i];
904 free_irq(sh_chan->irq, sh_chan);
906 list_del(&sh_chan->common.device_node);
908 shdev->chan[i] = NULL;
911 shdev->common.chancnt = 0;
914 static int __init sh_dmae_probe(struct platform_device *pdev)
916 struct sh_dmae_pdata *pdata = pdev->dev.platform_data;
917 unsigned long irqflags = IRQF_DISABLED,
918 chan_flag[SH_DMAC_MAX_CHANNELS] = {};
919 int errirq, chan_irq[SH_DMAC_MAX_CHANNELS];
920 int err, i, irq_cnt = 0, irqres = 0;
921 struct sh_dmae_device *shdev;
922 struct resource *chan, *dmars, *errirq_res, *chanirq_res;
924 /* get platform data */
925 if (!pdata || !pdata->channel_num)
928 chan = platform_get_resource(pdev, IORESOURCE_MEM, 0);
929 /* DMARS area is optional, if absent, this controller cannot do slave DMA */
930 dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1);
933 * 1. there always must be at least one IRQ IO-resource. On SH4 it is
934 * the error IRQ, in which case it is the only IRQ in this resource:
935 * start == end. If it is the only IRQ resource, all channels also
937 * 2. DMA channel IRQ resources can be specified one per resource or in
938 * ranges (start != end)
939 * 3. iff all events (channels and, optionally, error) on this
940 * controller use the same IRQ, only one IRQ resource can be
941 * specified, otherwise there must be one IRQ per channel, even if
942 * some of them are equal
943 * 4. if all IRQs on this controller are equal or if some specific IRQs
944 * specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be
945 * requested with the IRQF_SHARED flag
947 errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
948 if (!chan || !errirq_res)
951 if (!request_mem_region(chan->start, resource_size(chan), pdev->name)) {
952 dev_err(&pdev->dev, "DMAC register region already claimed\n");
956 if (dmars && !request_mem_region(dmars->start, resource_size(dmars), pdev->name)) {
957 dev_err(&pdev->dev, "DMAC DMARS region already claimed\n");
963 shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL);
965 dev_err(&pdev->dev, "Not enough memory\n");
969 shdev->chan_reg = ioremap(chan->start, resource_size(chan));
970 if (!shdev->chan_reg)
973 shdev->dmars = ioremap(dmars->start, resource_size(dmars));
979 shdev->pdata = pdata;
981 /* reset dma controller */
982 err = sh_dmae_rst(shdev);
986 INIT_LIST_HEAD(&shdev->common.channels);
988 dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask);
990 dma_cap_set(DMA_SLAVE, shdev->common.cap_mask);
992 shdev->common.device_alloc_chan_resources
993 = sh_dmae_alloc_chan_resources;
994 shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources;
995 shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy;
996 shdev->common.device_is_tx_complete = sh_dmae_is_complete;
997 shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending;
999 /* Compulsory for DMA_SLAVE fields */
1000 shdev->common.device_prep_slave_sg = sh_dmae_prep_slave_sg;
1001 shdev->common.device_terminate_all = sh_dmae_terminate_all;
1003 shdev->common.dev = &pdev->dev;
1004 /* Default transfer size of 32 bytes requires 32-byte alignment */
1005 shdev->common.copy_align = LOG2_DEFAULT_XFER_SIZE;
1007 #if defined(CONFIG_CPU_SH4)
1008 chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1);
1011 chanirq_res = errirq_res;
1015 if (chanirq_res == errirq_res ||
1016 (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE)
1017 irqflags = IRQF_SHARED;
1019 errirq = errirq_res->start;
1021 err = request_irq(errirq, sh_dmae_err, irqflags,
1022 "DMAC Address Error", shdev);
1025 "DMA failed requesting irq #%d, error %d\n",
1031 chanirq_res = errirq_res;
1032 #endif /* CONFIG_CPU_SH4 */
1034 if (chanirq_res->start == chanirq_res->end &&
1035 !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) {
1036 /* Special case - all multiplexed */
1037 for (; irq_cnt < pdata->channel_num; irq_cnt++) {
1038 chan_irq[irq_cnt] = chanirq_res->start;
1039 chan_flag[irq_cnt] = IRQF_SHARED;
1043 for (i = chanirq_res->start; i <= chanirq_res->end; i++) {
1044 if ((errirq_res->flags & IORESOURCE_BITS) ==
1045 IORESOURCE_IRQ_SHAREABLE)
1046 chan_flag[irq_cnt] = IRQF_SHARED;
1048 chan_flag[irq_cnt] = IRQF_DISABLED;
1050 "Found IRQ %d for channel %d\n",
1052 chan_irq[irq_cnt++] = i;
1054 chanirq_res = platform_get_resource(pdev,
1055 IORESOURCE_IRQ, ++irqres);
1056 } while (irq_cnt < pdata->channel_num && chanirq_res);
1059 if (irq_cnt < pdata->channel_num)
1062 /* Create DMA Channel */
1063 for (i = 0; i < pdata->channel_num; i++) {
1064 err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]);
1066 goto chan_probe_err;
1069 platform_set_drvdata(pdev, shdev);
1070 dma_async_device_register(&shdev->common);
1075 sh_dmae_chan_remove(shdev);
1077 #if defined(CONFIG_CPU_SH4)
1078 free_irq(errirq, shdev);
1083 iounmap(shdev->dmars);
1085 iounmap(shdev->chan_reg);
1090 release_mem_region(dmars->start, resource_size(dmars));
1092 release_mem_region(chan->start, resource_size(chan));
1097 static int __exit sh_dmae_remove(struct platform_device *pdev)
1099 struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
1100 struct resource *res;
1101 int errirq = platform_get_irq(pdev, 0);
1103 dma_async_device_unregister(&shdev->common);
1106 free_irq(errirq, shdev);
1108 /* channel data remove */
1109 sh_dmae_chan_remove(shdev);
1112 iounmap(shdev->dmars);
1113 iounmap(shdev->chan_reg);
1117 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1119 release_mem_region(res->start, resource_size(res));
1120 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1122 release_mem_region(res->start, resource_size(res));
1127 static void sh_dmae_shutdown(struct platform_device *pdev)
1129 struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
1130 sh_dmae_ctl_stop(shdev);
1133 static struct platform_driver sh_dmae_driver = {
1134 .remove = __exit_p(sh_dmae_remove),
1135 .shutdown = sh_dmae_shutdown,
1137 .name = "sh-dma-engine",
1141 static int __init sh_dmae_init(void)
1143 return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe);
1145 module_init(sh_dmae_init);
1147 static void __exit sh_dmae_exit(void)
1149 platform_driver_unregister(&sh_dmae_driver);
1151 module_exit(sh_dmae_exit);
1153 MODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>");
1154 MODULE_DESCRIPTION("Renesas SH DMA Engine driver");
1155 MODULE_LICENSE("GPL");