SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/dma/ioat_dma.c

   1 /*
   2  * Intel I/OAT DMA Linux driver
   3  * Copyright(c) 2004 - 2007 Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  */
  22
  23 /*
  24  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
  25  * copy operations.
  26  */
  27
  28 #include <linux/init.h>
  29 #include <linux/module.h>
  30 #include <linux/pci.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/dmaengine.h>
  33 #include <linux/delay.h>
  34 #include <linux/dma-mapping.h>
  35 #include "ioatdma.h"
  36 #include "ioatdma_registers.h"
  37 #include "ioatdma_hw.h"
  38
  39 #define INITIAL_IOAT_DESC_COUNT 128
  40
  41 #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
  42 #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
  43 #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
  44 #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
  45
  46 /* internal functions */
  47 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);
  48 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
  49
  50 static struct ioat_dma_chan *ioat_lookup_chan_by_index(struct ioatdma_device *device,
  51                                                        int index)
  52 {
  53         return device->idx[index];
  54 }
  55
  56 /**
  57  * ioat_dma_do_interrupt - handler used for single vector interrupt mode
  58  * @irq: interrupt id
  59  * @data: interrupt data
  60  */
  61 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
  62 {
  63         struct ioatdma_device *instance = data;
  64         struct ioat_dma_chan *ioat_chan;
  65         unsigned long attnstatus;
  66         int bit;
  67         u8 intrctrl;
  68
  69         intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
  70
  71         if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
  72                 return IRQ_NONE;
  73
  74         if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
  75                 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  76                 return IRQ_NONE;
  77         }
  78
  79         attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
  80         for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
  81                 ioat_chan = ioat_lookup_chan_by_index(instance, bit);
  82                 tasklet_schedule(&ioat_chan->cleanup_task);
  83         }
  84
  85         writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  86         return IRQ_HANDLED;
  87 }
  88
  89 /**
  90  * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
  91  * @irq: interrupt id
  92  * @data: interrupt data
  93  */
  94 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
  95 {
  96         struct ioat_dma_chan *ioat_chan = data;
  97
  98         tasklet_schedule(&ioat_chan->cleanup_task);
  99
 100         return IRQ_HANDLED;
 101 }
 102
 103 static void ioat_dma_cleanup_tasklet(unsigned long data);
 104
 105 /**
 106  * ioat_dma_enumerate_channels - find and initialize the device's channels
 107  * @device: the device to be enumerated
 108  */
 109 static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
 110 {
 111         u8 xfercap_scale;
 112         u32 xfercap;
 113         int i;
 114         struct ioat_dma_chan *ioat_chan;
 115
 116         device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
 117         xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
 118         xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
 119
 120         for (i = 0; i < device->common.chancnt; i++) {
 121                 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
 122                 if (!ioat_chan) {
 123                         device->common.chancnt = i;
 124                         break;
 125                 }
 126
 127                 ioat_chan->device = device;
 128                 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
 129                 ioat_chan->xfercap = xfercap;
 130                 spin_lock_init(&ioat_chan->cleanup_lock);
 131                 spin_lock_init(&ioat_chan->desc_lock);
 132                 INIT_LIST_HEAD(&ioat_chan->free_desc);
 133                 INIT_LIST_HEAD(&ioat_chan->used_desc);
 134                 /* This should be made common somewhere in dmaengine.c */
 135                 ioat_chan->common.device = &device->common;
 136                 list_add_tail(&ioat_chan->common.device_node,
 137                               &device->common.channels);
 138                 device->idx[i] = ioat_chan;
 139                 tasklet_init(&ioat_chan->cleanup_task,
 140                              ioat_dma_cleanup_tasklet,
 141                              (unsigned long) ioat_chan);
 142                 tasklet_disable(&ioat_chan->cleanup_task);
 143         }
 144         return device->common.chancnt;
 145 }
 146
 147 static void ioat_set_src(dma_addr_t addr,
 148                          struct dma_async_tx_descriptor *tx,
 149                          int index)
 150 {
 151         struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
 152         struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
 153
 154         pci_unmap_addr_set(desc, src, addr);
 155
 156         list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
 157                 iter->hw->src_addr = addr;
 158                 addr += ioat_chan->xfercap;
 159         }
 160
 161 }
 162
 163 static void ioat_set_dest(dma_addr_t addr,
 164                           struct dma_async_tx_descriptor *tx,
 165                           int index)
 166 {
 167         struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
 168         struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
 169
 170         pci_unmap_addr_set(desc, dst, addr);
 171
 172         list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
 173                 iter->hw->dst_addr = addr;
 174                 addr += ioat_chan->xfercap;
 175         }
 176 }
 177
 178 static dma_cookie_t ioat_tx_submit(struct dma_async_tx_descriptor *tx)
 179 {
 180         struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
 181         struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
 182         int append = 0;
 183         dma_cookie_t cookie;
 184         struct ioat_desc_sw *group_start;
 185
 186         group_start = list_entry(desc->async_tx.tx_list.next,
 187                                  struct ioat_desc_sw, node);
 188         spin_lock_bh(&ioat_chan->desc_lock);
 189         /* cookie incr and addition to used_list must be atomic */
 190         cookie = ioat_chan->common.cookie;
 191         cookie++;
 192         if (cookie < 0)
 193                 cookie = 1;
 194         ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
 195
 196         /* write address into NextDescriptor field of last desc in chain */
 197         to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
 198                                                 group_start->async_tx.phys;
 199         list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
 200
 201         ioat_chan->pending += desc->tx_cnt;
 202         if (ioat_chan->pending >= 4) {
 203                 append = 1;
 204                 ioat_chan->pending = 0;
 205         }
 206         spin_unlock_bh(&ioat_chan->desc_lock);
 207
 208         if (append)
 209                 writeb(IOAT_CHANCMD_APPEND,
 210                         ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
 211
 212         return cookie;
 213 }
 214
 215 static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
 216                                         struct ioat_dma_chan *ioat_chan,
 217                                         gfp_t flags)
 218 {
 219         struct ioat_dma_descriptor *desc;
 220         struct ioat_desc_sw *desc_sw;
 221         struct ioatdma_device *ioatdma_device;
 222         dma_addr_t phys;
 223
 224         ioatdma_device = to_ioatdma_device(ioat_chan->common.device);
 225         desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
 226         if (unlikely(!desc))
 227                 return NULL;
 228
 229         desc_sw = kzalloc(sizeof(*desc_sw), flags);
 230         if (unlikely(!desc_sw)) {
 231                 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
 232                 return NULL;
 233         }
 234
 235         memset(desc, 0, sizeof(*desc));
 236         dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
 237         desc_sw->async_tx.tx_set_src = ioat_set_src;
 238         desc_sw->async_tx.tx_set_dest = ioat_set_dest;
 239         desc_sw->async_tx.tx_submit = ioat_tx_submit;
 240         INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
 241         desc_sw->hw = desc;
 242         desc_sw->async_tx.phys = phys;
 243
 244         return desc_sw;
 245 }
 246
 247 /* returns the actual number of allocated descriptors */
 248 static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
 249 {
 250         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 251         struct ioat_desc_sw *desc = NULL;
 252         u16 chanctrl;
 253         u32 chanerr;
 254         int i;
 255         LIST_HEAD(tmp_list);
 256
 257         /* have we already been set up? */
 258         if (!list_empty(&ioat_chan->free_desc))
 259                 return INITIAL_IOAT_DESC_COUNT;
 260
 261         /* Setup register to interrupt and write completion status on error */
 262         chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
 263                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 264                 IOAT_CHANCTRL_ERR_COMPLETION_EN;
 265         writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
 266
 267         chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 268         if (chanerr) {
 269                 dev_err(&ioat_chan->device->pdev->dev,
 270                         "CHANERR = %x, clearing\n", chanerr);
 271                 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 272         }
 273
 274         /* Allocate descriptors */
 275         for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
 276                 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
 277                 if (!desc) {
 278                         dev_err(&ioat_chan->device->pdev->dev,
 279                                 "Only %d initial descriptors\n", i);
 280                         break;
 281                 }
 282                 list_add_tail(&desc->node, &tmp_list);
 283         }
 284         spin_lock_bh(&ioat_chan->desc_lock);
 285         list_splice(&tmp_list, &ioat_chan->free_desc);
 286         spin_unlock_bh(&ioat_chan->desc_lock);
 287
 288         /* allocate a completion writeback area */
 289         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
 290         ioat_chan->completion_virt =
 291                 pci_pool_alloc(ioat_chan->device->completion_pool,
 292                                GFP_KERNEL,
 293                                &ioat_chan->completion_addr);
 294         memset(ioat_chan->completion_virt, 0,
 295                sizeof(*ioat_chan->completion_virt));
 296         writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
 297                ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
 298         writel(((u64) ioat_chan->completion_addr) >> 32,
 299                ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
 300
 301         tasklet_enable(&ioat_chan->cleanup_task);
 302         ioat_dma_start_null_desc(ioat_chan);
 303         return i;
 304 }
 305
 306 static void ioat_dma_free_chan_resources(struct dma_chan *chan)
 307 {
 308         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 309         struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device);
 310         struct ioat_desc_sw *desc, *_desc;
 311         int in_use_descs = 0;
 312
 313         tasklet_disable(&ioat_chan->cleanup_task);
 314         ioat_dma_memcpy_cleanup(ioat_chan);
 315
 316         /* Delay 100ms after reset to allow internal DMA logic to quiesce
 317          * before removing DMA descriptor resources.
 318          */
 319         writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
 320         mdelay(100);
 321
 322         spin_lock_bh(&ioat_chan->desc_lock);
 323         list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
 324                 in_use_descs++;
 325                 list_del(&desc->node);
 326                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 327                               desc->async_tx.phys);
 328                 kfree(desc);
 329         }
 330         list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
 331                 list_del(&desc->node);
 332                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 333                               desc->async_tx.phys);
 334                 kfree(desc);
 335         }
 336         spin_unlock_bh(&ioat_chan->desc_lock);
 337
 338         pci_pool_free(ioatdma_device->completion_pool,
 339                       ioat_chan->completion_virt,
 340                       ioat_chan->completion_addr);
 341
 342         /* one is ok since we left it on there on purpose */
 343         if (in_use_descs > 1)
 344                 dev_err(&ioat_chan->device->pdev->dev,
 345                         "Freeing %d in use descriptors!\n",
 346                         in_use_descs - 1);
 347
 348         ioat_chan->last_completion = ioat_chan->completion_addr = 0;
 349         ioat_chan->pending = 0;
 350 }
 351 /**
 352  * ioat_dma_get_next_descriptor - return the next available descriptor
 353  * @ioat_chan: IOAT DMA channel handle
 354  *
 355  * Gets the next descriptor from the chain, and must be called with the
 356  * channel's desc_lock held.  Allocates more descriptors if the channel
 357  * has run out.
 358  */
 359 static struct ioat_desc_sw *ioat_dma_get_next_descriptor(
 360                                                 struct ioat_dma_chan *ioat_chan)
 361 {
 362         struct ioat_desc_sw *new = NULL;
 363
 364         if (!list_empty(&ioat_chan->free_desc)) {
 365                 new = to_ioat_desc(ioat_chan->free_desc.next);
 366                 list_del(&new->node);
 367         } else {
 368                 /* try to get another desc */
 369                 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
 370                 /* will this ever happen? */
 371                 /* TODO add upper limit on these */
 372                 BUG_ON(!new);
 373         }
 374
 375         prefetch(new->hw);
 376         return new;
 377 }
 378
 379 static struct dma_async_tx_descriptor *ioat_dma_prep_memcpy(
 380                                                 struct dma_chan *chan,
 381                                                 size_t len,
 382                                                 int int_en)
 383 {
 384         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 385         struct ioat_desc_sw *first, *prev, *new;
 386         LIST_HEAD(new_chain);
 387         u32 copy;
 388         size_t orig_len;
 389         int desc_count = 0;
 390
 391         if (!len)
 392                 return NULL;
 393
 394         orig_len = len;
 395
 396         first = NULL;
 397         prev = NULL;
 398
 399         spin_lock_bh(&ioat_chan->desc_lock);
 400         while (len) {
 401                 new = ioat_dma_get_next_descriptor(ioat_chan);
 402                 copy = min((u32) len, ioat_chan->xfercap);
 403
 404                 new->hw->size = copy;
 405                 new->hw->ctl = 0;
 406                 new->async_tx.cookie = 0;
 407                 new->async_tx.ack = 1;
 408
 409                 /* chain together the physical address list for the HW */
 410                 if (!first)
 411                         first = new;
 412                 else
 413                         prev->hw->next = (u64) new->async_tx.phys;
 414
 415                 prev = new;
 416                 len  -= copy;
 417                 list_add_tail(&new->node, &new_chain);
 418                 desc_count++;
 419         }
 420
 421         list_splice(&new_chain, &new->async_tx.tx_list);
 422
 423         new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
 424         new->hw->next = 0;
 425         new->tx_cnt = desc_count;
 426         new->async_tx.ack = 0; /* client is in control of this ack */
 427         new->async_tx.cookie = -EBUSY;
 428
 429         pci_unmap_len_set(new, len, orig_len);
 430         spin_unlock_bh(&ioat_chan->desc_lock);
 431
 432         return new ? &new->async_tx : NULL;
 433 }
 434
 435 /**
 436  * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
 437  *                                 descriptors to hw
 438  * @chan: DMA channel handle
 439  */
 440 static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
 441 {
 442         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 443
 444         if (ioat_chan->pending != 0) {
 445                 ioat_chan->pending = 0;
 446                 writeb(IOAT_CHANCMD_APPEND,
 447                        ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
 448         }
 449 }
 450
 451 static void ioat_dma_cleanup_tasklet(unsigned long data)
 452 {
 453         struct ioat_dma_chan *chan = (void *)data;
 454         ioat_dma_memcpy_cleanup(chan);
 455         writew(IOAT_CHANCTRL_INT_DISABLE,
 456                chan->reg_base + IOAT_CHANCTRL_OFFSET);
 457 }
 458
 459 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan)
 460 {
 461         unsigned long phys_complete;
 462         struct ioat_desc_sw *desc, *_desc;
 463         dma_cookie_t cookie = 0;
 464
 465         prefetch(ioat_chan->completion_virt);
 466
 467         if (!spin_trylock(&ioat_chan->cleanup_lock))
 468                 return;
 469
 470         /* The completion writeback can happen at any time,
 471            so reads by the driver need to be atomic operations
 472            The descriptor physical addresses are limited to 32-bits
 473            when the CPU can only do a 32-bit mov */
 474
 475 #if (BITS_PER_LONG == 64)
 476         phys_complete =
 477         ioat_chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
 478 #else
 479         phys_complete = ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
 480 #endif
 481
 482         if ((ioat_chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
 483                                 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
 484                 dev_err(&ioat_chan->device->pdev->dev,
 485                         "Channel halted, chanerr = %x\n",
 486                         readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET));
 487
 488                 /* TODO do something to salvage the situation */
 489         }
 490
 491         if (phys_complete == ioat_chan->last_completion) {
 492                 spin_unlock(&ioat_chan->cleanup_lock);
 493                 return;
 494         }
 495
 496         cookie = 0;
 497         spin_lock_bh(&ioat_chan->desc_lock);
 498         list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
 499
 500                 /*
 501                  * Incoming DMA requests may use multiple descriptors, due to
 502                  * exceeding xfercap, perhaps. If so, only the last one will
 503                  * have a cookie, and require unmapping.
 504                  */
 505                 if (desc->async_tx.cookie) {
 506                         cookie = desc->async_tx.cookie;
 507
 508                         /*
 509                          * yes we are unmapping both _page and _single alloc'd
 510                          * regions with unmap_page. Is this *really* that bad?
 511                          */
 512                         pci_unmap_page(ioat_chan->device->pdev,
 513                                         pci_unmap_addr(desc, dst),
 514                                         pci_unmap_len(desc, len),
 515                                         PCI_DMA_FROMDEVICE);
 516                         pci_unmap_page(ioat_chan->device->pdev,
 517                                         pci_unmap_addr(desc, src),
 518                                         pci_unmap_len(desc, len),
 519                                         PCI_DMA_TODEVICE);
 520                 }
 521
 522                 if (desc->async_tx.phys != phys_complete) {
 523                         /*
 524                          * a completed entry, but not the last, so cleanup
 525                          * if the client is done with the descriptor
 526                          */
 527                         if (desc->async_tx.ack) {
 528                                 list_del(&desc->node);
 529                                 list_add_tail(&desc->node,
 530                                               &ioat_chan->free_desc);
 531                         } else
 532                                 desc->async_tx.cookie = 0;
 533                 } else {
 534                         /*
 535                          * last used desc. Do not remove, so we can append from
 536                          * it, but don't look at it next time, either
 537                          */
 538                         desc->async_tx.cookie = 0;
 539
 540                         /* TODO check status bits? */
 541                         break;
 542                 }
 543         }
 544
 545         spin_unlock_bh(&ioat_chan->desc_lock);
 546
 547         ioat_chan->last_completion = phys_complete;
 548         if (cookie != 0)
 549                 ioat_chan->completed_cookie = cookie;
 550
 551         spin_unlock(&ioat_chan->cleanup_lock);
 552 }
 553
 554 static void ioat_dma_dependency_added(struct dma_chan *chan)
 555 {
 556         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 557         spin_lock_bh(&ioat_chan->desc_lock);
 558         if (ioat_chan->pending == 0) {
 559                 spin_unlock_bh(&ioat_chan->desc_lock);
 560                 ioat_dma_memcpy_cleanup(ioat_chan);
 561         } else
 562                 spin_unlock_bh(&ioat_chan->desc_lock);
 563 }
 564
 565 /**
 566  * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
 567  * @chan: IOAT DMA channel handle
 568  * @cookie: DMA transaction identifier
 569  * @done: if not %NULL, updated with last completed transaction
 570  * @used: if not %NULL, updated with last used transaction
 571  */
 572 static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
 573                                             dma_cookie_t cookie,
 574                                             dma_cookie_t *done,
 575                                             dma_cookie_t *used)
 576 {
 577         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 578         dma_cookie_t last_used;
 579         dma_cookie_t last_complete;
 580         enum dma_status ret;
 581
 582         last_used = chan->cookie;
 583         last_complete = ioat_chan->completed_cookie;
 584
 585         if (done)
 586                 *done = last_complete;
 587         if (used)
 588                 *used = last_used;
 589
 590         ret = dma_async_is_complete(cookie, last_complete, last_used);
 591         if (ret == DMA_SUCCESS)
 592                 return ret;
 593
 594         ioat_dma_memcpy_cleanup(ioat_chan);
 595
 596         last_used = chan->cookie;
 597         last_complete = ioat_chan->completed_cookie;
 598
 599         if (done)
 600                 *done = last_complete;
 601         if (used)
 602                 *used = last_used;
 603
 604         return dma_async_is_complete(cookie, last_complete, last_used);
 605 }
 606
 607 /* PCI API */
 608
 609 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan)
 610 {
 611         struct ioat_desc_sw *desc;
 612
 613         spin_lock_bh(&ioat_chan->desc_lock);
 614
 615         desc = ioat_dma_get_next_descriptor(ioat_chan);
 616         desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
 617         desc->hw->next = 0;
 618         desc->async_tx.ack = 1;
 619
 620         list_add_tail(&desc->node, &ioat_chan->used_desc);
 621         spin_unlock_bh(&ioat_chan->desc_lock);
 622
 623         writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
 624                ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
 625         writel(((u64) desc->async_tx.phys) >> 32,
 626                ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
 627
 628         writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
 629 }
 630
 631 /*
 632  * Perform a IOAT transaction to verify the HW works.
 633  */
 634 #define IOAT_TEST_SIZE 2000
 635
 636 /**
 637  * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
 638  * @device: device to be tested
 639  */
 640 static int ioat_dma_self_test(struct ioatdma_device *device)
 641 {
 642         int i;
 643         u8 *src;
 644         u8 *dest;
 645         struct dma_chan *dma_chan;
 646         struct dma_async_tx_descriptor *tx = NULL;
 647         dma_addr_t addr;
 648         dma_cookie_t cookie;
 649         int err = 0;
 650
 651         src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
 652         if (!src)
 653                 return -ENOMEM;
 654         dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
 655         if (!dest) {
 656                 kfree(src);
 657                 return -ENOMEM;
 658         }
 659
 660         /* Fill in src buffer */
 661         for (i = 0; i < IOAT_TEST_SIZE; i++)
 662                 src[i] = (u8)i;
 663
 664         /* Start copy, using first DMA channel */
 665         dma_chan = container_of(device->common.channels.next,
 666                                 struct dma_chan,
 667                                 device_node);
 668         if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
 669                 dev_err(&device->pdev->dev,
 670                         "selftest cannot allocate chan resource\n");
 671                 err = -ENODEV;
 672                 goto out;
 673         }
 674
 675         tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
 676         if (!tx) {
 677                 dev_err(&device->pdev->dev,
 678                         "Self-test prep failed, disabling\n");
 679                 err = -ENODEV;
 680                 goto free_resources;
 681         }
 682
 683         async_tx_ack(tx);
 684         addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
 685                         DMA_TO_DEVICE);
 686         ioat_set_src(addr, tx, 0);
 687         addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
 688                         DMA_FROM_DEVICE);
 689         ioat_set_dest(addr, tx, 0);
 690         cookie = ioat_tx_submit(tx);
 691         ioat_dma_memcpy_issue_pending(dma_chan);
 692         msleep(1);
 693
 694         if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
 695                 dev_err(&device->pdev->dev,
 696                         "Self-test copy timed out, disabling\n");
 697                 err = -ENODEV;
 698                 goto free_resources;
 699         }
 700         if (memcmp(src, dest, IOAT_TEST_SIZE)) {
 701                 dev_err(&device->pdev->dev,
 702                         "Self-test copy failed compare, disabling\n");
 703                 err = -ENODEV;
 704                 goto free_resources;
 705         }
 706
 707 free_resources:
 708         ioat_dma_free_chan_resources(dma_chan);
 709 out:
 710         kfree(src);
 711         kfree(dest);
 712         return err;
 713 }
 714
 715 static char ioat_interrupt_style[32] = "msix";
 716 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
 717                     sizeof(ioat_interrupt_style), 0644);
 718 MODULE_PARM_DESC(ioat_interrupt_style,
 719                  "set ioat interrupt style: msix (default), "
 720                  "msix-single-vector, msi, intx)");
 721
 722 /**
 723  * ioat_dma_setup_interrupts - setup interrupt handler
 724  * @device: ioat device
 725  */
 726 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
 727 {
 728         struct ioat_dma_chan *ioat_chan;
 729         int err, i, j, msixcnt;
 730         u8 intrctrl = 0;
 731
 732         if (!strcmp(ioat_interrupt_style, "msix"))
 733                 goto msix;
 734         if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
 735                 goto msix_single_vector;
 736         if (!strcmp(ioat_interrupt_style, "msi"))
 737                 goto msi;
 738         if (!strcmp(ioat_interrupt_style, "intx"))
 739                 goto intx;
 740         dev_err(&device->pdev->dev, "invalid ioat_interrupt_style %s\n",
 741                 ioat_interrupt_style);
 742         goto err_no_irq;
 743
 744 msix:
 745         /* The number of MSI-X vectors should equal the number of channels */
 746         msixcnt = device->common.chancnt;
 747         for (i = 0; i < msixcnt; i++)
 748                 device->msix_entries[i].entry = i;
 749
 750         err = pci_enable_msix(device->pdev, device->msix_entries, msixcnt);
 751         if (err < 0)
 752                 goto msi;
 753         if (err > 0)
 754                 goto msix_single_vector;
 755
 756         for (i = 0; i < msixcnt; i++) {
 757                 ioat_chan = ioat_lookup_chan_by_index(device, i);
 758                 err = request_irq(device->msix_entries[i].vector,
 759                                   ioat_dma_do_interrupt_msix,
 760                                   0, "ioat-msix", ioat_chan);
 761                 if (err) {
 762                         for (j = 0; j < i; j++) {
 763                                 ioat_chan =
 764                                         ioat_lookup_chan_by_index(device, j);
 765                                 free_irq(device->msix_entries[j].vector,
 766                                          ioat_chan);
 767                         }
 768                         goto msix_single_vector;
 769                 }
 770         }
 771         intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
 772         device->irq_mode = msix_multi_vector;
 773         goto done;
 774
 775 msix_single_vector:
 776         device->msix_entries[0].entry = 0;
 777         err = pci_enable_msix(device->pdev, device->msix_entries, 1);
 778         if (err)
 779                 goto msi;
 780
 781         err = request_irq(device->msix_entries[0].vector, ioat_dma_do_interrupt,
 782                           0, "ioat-msix", device);
 783         if (err) {
 784                 pci_disable_msix(device->pdev);
 785                 goto msi;
 786         }
 787         device->irq_mode = msix_single_vector;
 788         goto done;
 789
 790 msi:
 791         err = pci_enable_msi(device->pdev);
 792         if (err)
 793                 goto intx;
 794
 795         err = request_irq(device->pdev->irq, ioat_dma_do_interrupt,
 796                           0, "ioat-msi", device);
 797         if (err) {
 798                 pci_disable_msi(device->pdev);
 799                 goto intx;
 800         }
 801         /*
 802          * CB 1.2 devices need a bit set in configuration space to enable MSI
 803          */
 804         if (device->version == IOAT_VER_1_2) {
 805                 u32 dmactrl;
 806                 pci_read_config_dword(device->pdev,
 807                                       IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
 808                 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
 809                 pci_write_config_dword(device->pdev,
 810                                        IOAT_PCI_DMACTRL_OFFSET, dmactrl);
 811         }
 812         device->irq_mode = msi;
 813         goto done;
 814
 815 intx:
 816         err = request_irq(device->pdev->irq, ioat_dma_do_interrupt,
 817                           IRQF_SHARED, "ioat-intx", device);
 818         if (err)
 819                 goto err_no_irq;
 820         device->irq_mode = intx;
 821
 822 done:
 823         intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
 824         writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
 825         return 0;
 826
 827 err_no_irq:
 828         /* Disable all interrupt generation */
 829         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
 830         dev_err(&device->pdev->dev, "no usable interrupts\n");
 831         device->irq_mode = none;
 832         return -1;
 833 }
 834
 835 /**
 836  * ioat_dma_remove_interrupts - remove whatever interrupts were set
 837  * @device: ioat device
 838  */
 839 static void ioat_dma_remove_interrupts(struct ioatdma_device *device)
 840 {
 841         struct ioat_dma_chan *ioat_chan;
 842         int i;
 843
 844         /* Disable all interrupt generation */
 845         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
 846
 847         switch (device->irq_mode) {
 848         case msix_multi_vector:
 849                 for (i = 0; i < device->common.chancnt; i++) {
 850                         ioat_chan = ioat_lookup_chan_by_index(device, i);
 851                         free_irq(device->msix_entries[i].vector, ioat_chan);
 852                 }
 853                 pci_disable_msix(device->pdev);
 854                 break;
 855         case msix_single_vector:
 856                 free_irq(device->msix_entries[0].vector, device);
 857                 pci_disable_msix(device->pdev);
 858                 break;
 859         case msi:
 860                 free_irq(device->pdev->irq, device);
 861                 pci_disable_msi(device->pdev);
 862                 break;
 863         case intx:
 864                 free_irq(device->pdev->irq, device);
 865                 break;
 866         case none:
 867                 dev_warn(&device->pdev->dev,
 868                          "call to %s without interrupts setup\n", __func__);
 869         }
 870         device->irq_mode = none;
 871 }
 872
 873 struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
 874                                       void __iomem *iobase)
 875 {
 876         int err;
 877         struct ioatdma_device *device;
 878
 879         device = kzalloc(sizeof(*device), GFP_KERNEL);
 880         if (!device) {
 881                 err = -ENOMEM;
 882                 goto err_kzalloc;
 883         }
 884         device->pdev = pdev;
 885         device->reg_base = iobase;
 886         device->version = readb(device->reg_base + IOAT_VER_OFFSET);
 887
 888         /* DMA coherent memory pool for DMA descriptor allocations */
 889         device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
 890                                            sizeof(struct ioat_dma_descriptor),
 891                                            64, 0);
 892         if (!device->dma_pool) {
 893                 err = -ENOMEM;
 894                 goto err_dma_pool;
 895         }
 896
 897         device->completion_pool = pci_pool_create("completion_pool", pdev,
 898                                                   sizeof(u64), SMP_CACHE_BYTES,
 899                                                   SMP_CACHE_BYTES);
 900         if (!device->completion_pool) {
 901                 err = -ENOMEM;
 902                 goto err_completion_pool;
 903         }
 904
 905         INIT_LIST_HEAD(&device->common.channels);
 906         ioat_dma_enumerate_channels(device);
 907
 908         dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
 909         device->common.device_alloc_chan_resources =
 910                                                 ioat_dma_alloc_chan_resources;
 911         device->common.device_free_chan_resources =
 912                                                 ioat_dma_free_chan_resources;
 913         device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
 914         device->common.device_is_tx_complete = ioat_dma_is_complete;
 915         device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
 916         device->common.device_dependency_added = ioat_dma_dependency_added;
 917         device->common.dev = &pdev->dev;
 918         dev_err(&device->pdev->dev,
 919                 "Intel(R) I/OAT DMA Engine found,"
 920                 " %d channels, device version 0x%02x, driver version %s\n",
 921                 device->common.chancnt, device->version, IOAT_DMA_VERSION);
 922
 923         err = ioat_dma_setup_interrupts(device);
 924         if (err)
 925                 goto err_setup_interrupts;
 926
 927         err = ioat_dma_self_test(device);
 928         if (err)
 929                 goto err_self_test;
 930
 931         dma_async_device_register(&device->common);
 932
 933         return device;
 934
 935 err_self_test:
 936         ioat_dma_remove_interrupts(device);
 937 err_setup_interrupts:
 938         pci_pool_destroy(device->completion_pool);
 939 err_completion_pool:
 940         pci_pool_destroy(device->dma_pool);
 941 err_dma_pool:
 942         kfree(device);
 943 err_kzalloc:
 944         dev_err(&device->pdev->dev,
 945                 "Intel(R) I/OAT DMA Engine initialization failed\n");
 946         return NULL;
 947 }
 948
 949 void ioat_dma_remove(struct ioatdma_device *device)
 950 {
 951         struct dma_chan *chan, *_chan;
 952         struct ioat_dma_chan *ioat_chan;
 953
 954         ioat_dma_remove_interrupts(device);
 955
 956         dma_async_device_unregister(&device->common);
 957
 958         pci_pool_destroy(device->dma_pool);
 959         pci_pool_destroy(device->completion_pool);
 960
 961         iounmap(device->reg_base);
 962         pci_release_regions(device->pdev);
 963         pci_disable_device(device->pdev);
 964
 965         list_for_each_entry_safe(chan, _chan,
 966                                  &device->common.channels, device_node) {
 967                 ioat_chan = to_ioat_chan(chan);
 968                 list_del(&chan->device_node);
 969                 kfree(ioat_chan);
 970         }
 971         kfree(device);
 972 }
 973