SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/dma/ioat/dma.c

   1 /*
   2  * Intel I/OAT DMA Linux driver
   3  * Copyright(c) 2004 - 2009 Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  */
  22
  23 /*
  24  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
  25  * copy operations.
  26  */
  27
  28 #include <linux/init.h>
  29 #include <linux/module.h>
  30 #include <linux/pci.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/dmaengine.h>
  33 #include <linux/delay.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/i7300_idle.h>
  37 #include "dma.h"
  38 #include "registers.h"
  39 #include "hw.h"
  40
  41 static int ioat_pending_level = 4;
  42 module_param(ioat_pending_level, int, 0644);
  43 MODULE_PARM_DESC(ioat_pending_level,
  44                  "high-water mark for pushing ioat descriptors (default: 4)");
  45
  46 static void ioat_dma_chan_reset_part2(struct work_struct *work);
  47 static void ioat_dma_chan_watchdog(struct work_struct *work);
  48
  49 /* internal functions */
  50 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);
  51 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
  52
  53 static struct ioat_desc_sw *
  54 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
  55 static struct ioat_desc_sw *
  56 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
  57
  58 static inline struct ioat_dma_chan *
  59 ioat_chan_by_index(struct ioatdma_device *device, int index)
  60 {
  61         return device->idx[index];
  62 }
  63
  64 /**
  65  * ioat_dma_do_interrupt - handler used for single vector interrupt mode
  66  * @irq: interrupt id
  67  * @data: interrupt data
  68  */
  69 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
  70 {
  71         struct ioatdma_device *instance = data;
  72         struct ioat_dma_chan *ioat_chan;
  73         unsigned long attnstatus;
  74         int bit;
  75         u8 intrctrl;
  76
  77         intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
  78
  79         if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
  80                 return IRQ_NONE;
  81
  82         if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
  83                 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  84                 return IRQ_NONE;
  85         }
  86
  87         attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
  88         for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
  89                 ioat_chan = ioat_chan_by_index(instance, bit);
  90                 tasklet_schedule(&ioat_chan->cleanup_task);
  91         }
  92
  93         writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  94         return IRQ_HANDLED;
  95 }
  96
  97 /**
  98  * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
  99  * @irq: interrupt id
 100  * @data: interrupt data
 101  */
 102 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
 103 {
 104         struct ioat_dma_chan *ioat_chan = data;
 105
 106         tasklet_schedule(&ioat_chan->cleanup_task);
 107
 108         return IRQ_HANDLED;
 109 }
 110
 111 static void ioat_dma_cleanup_tasklet(unsigned long data);
 112
 113 /**
 114  * ioat_dma_enumerate_channels - find and initialize the device's channels
 115  * @device: the device to be enumerated
 116  */
 117 static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
 118 {
 119         u8 xfercap_scale;
 120         u32 xfercap;
 121         int i;
 122         struct ioat_dma_chan *ioat_chan;
 123         struct device *dev = &device->pdev->dev;
 124         struct dma_device *dma = &device->common;
 125
 126         INIT_LIST_HEAD(&dma->channels);
 127         dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
 128         xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
 129         xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
 130
 131 #ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
 132         if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
 133                 dma->chancnt--;
 134 #endif
 135         for (i = 0; i < dma->chancnt; i++) {
 136                 ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
 137                 if (!ioat_chan) {
 138                         dma->chancnt = i;
 139                         break;
 140                 }
 141
 142                 ioat_chan->device = device;
 143                 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
 144                 ioat_chan->xfercap = xfercap;
 145                 ioat_chan->desccount = 0;
 146                 INIT_DELAYED_WORK(&ioat_chan->work, ioat_dma_chan_reset_part2);
 147                 spin_lock_init(&ioat_chan->cleanup_lock);
 148                 spin_lock_init(&ioat_chan->desc_lock);
 149                 INIT_LIST_HEAD(&ioat_chan->free_desc);
 150                 INIT_LIST_HEAD(&ioat_chan->used_desc);
 151                 /* This should be made common somewhere in dmaengine.c */
 152                 ioat_chan->common.device = &device->common;
 153                 list_add_tail(&ioat_chan->common.device_node, &dma->channels);
 154                 device->idx[i] = ioat_chan;
 155                 tasklet_init(&ioat_chan->cleanup_task,
 156                              ioat_dma_cleanup_tasklet,
 157                              (unsigned long) ioat_chan);
 158                 tasklet_disable(&ioat_chan->cleanup_task);
 159         }
 160         return dma->chancnt;
 161 }
 162
 163 /**
 164  * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
 165  *                                 descriptors to hw
 166  * @chan: DMA channel handle
 167  */
 168 static inline void
 169 __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat_chan)
 170 {
 171         ioat_chan->pending = 0;
 172         writeb(IOAT_CHANCMD_APPEND, ioat_chan->reg_base + IOAT1_CHANCMD_OFFSET);
 173 }
 174
 175 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
 176 {
 177         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 178
 179         if (ioat_chan->pending > 0) {
 180                 spin_lock_bh(&ioat_chan->desc_lock);
 181                 __ioat1_dma_memcpy_issue_pending(ioat_chan);
 182                 spin_unlock_bh(&ioat_chan->desc_lock);
 183         }
 184 }
 185
 186 static inline void
 187 __ioat2_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat_chan)
 188 {
 189         ioat_chan->pending = 0;
 190         writew(ioat_chan->dmacount,
 191                ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
 192 }
 193
 194 static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
 195 {
 196         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 197
 198         if (ioat_chan->pending > 0) {
 199                 spin_lock_bh(&ioat_chan->desc_lock);
 200                 __ioat2_dma_memcpy_issue_pending(ioat_chan);
 201                 spin_unlock_bh(&ioat_chan->desc_lock);
 202         }
 203 }
 204
 205
 206 /**
 207  * ioat_dma_chan_reset_part2 - reinit the channel after a reset
 208  */
 209 static void ioat_dma_chan_reset_part2(struct work_struct *work)
 210 {
 211         struct ioat_dma_chan *ioat_chan =
 212                 container_of(work, struct ioat_dma_chan, work.work);
 213         struct ioat_desc_sw *desc;
 214
 215         spin_lock_bh(&ioat_chan->cleanup_lock);
 216         spin_lock_bh(&ioat_chan->desc_lock);
 217
 218         ioat_chan->completion_virt->low = 0;
 219         ioat_chan->completion_virt->high = 0;
 220         ioat_chan->pending = 0;
 221
 222         /*
 223          * count the descriptors waiting, and be sure to do it
 224          * right for both the CB1 line and the CB2 ring
 225          */
 226         ioat_chan->dmacount = 0;
 227         if (ioat_chan->used_desc.prev) {
 228                 desc = to_ioat_desc(ioat_chan->used_desc.prev);
 229                 do {
 230                         ioat_chan->dmacount++;
 231                         desc = to_ioat_desc(desc->node.next);
 232                 } while (&desc->node != ioat_chan->used_desc.next);
 233         }
 234
 235         /*
 236          * write the new starting descriptor address
 237          * this puts channel engine into ARMED state
 238          */
 239         desc = to_ioat_desc(ioat_chan->used_desc.prev);
 240         switch (ioat_chan->device->version) {
 241         case IOAT_VER_1_2:
 242                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
 243                        ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
 244                 writel(((u64) desc->txd.phys) >> 32,
 245                        ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
 246
 247                 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
 248                         + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
 249                 break;
 250         case IOAT_VER_2_0:
 251                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
 252                        ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
 253                 writel(((u64) desc->txd.phys) >> 32,
 254                        ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
 255
 256                 /* tell the engine to go with what's left to be done */
 257                 writew(ioat_chan->dmacount,
 258                        ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
 259
 260                 break;
 261         }
 262         dev_err(to_dev(ioat_chan),
 263                 "chan%d reset - %d descs waiting, %d total desc\n",
 264                 chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
 265
 266         spin_unlock_bh(&ioat_chan->desc_lock);
 267         spin_unlock_bh(&ioat_chan->cleanup_lock);
 268 }
 269
 270 /**
 271  * ioat_dma_reset_channel - restart a channel
 272  * @ioat_chan: IOAT DMA channel handle
 273  */
 274 static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat_chan)
 275 {
 276         u32 chansts, chanerr;
 277
 278         if (!ioat_chan->used_desc.prev)
 279                 return;
 280
 281         chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 282         chansts = (ioat_chan->completion_virt->low
 283                                         & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
 284         if (chanerr) {
 285                 dev_err(to_dev(ioat_chan),
 286                         "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
 287                         chan_num(ioat_chan), chansts, chanerr);
 288                 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 289         }
 290
 291         /*
 292          * whack it upside the head with a reset
 293          * and wait for things to settle out.
 294          * force the pending count to a really big negative
 295          * to make sure no one forces an issue_pending
 296          * while we're waiting.
 297          */
 298
 299         spin_lock_bh(&ioat_chan->desc_lock);
 300         ioat_chan->pending = INT_MIN;
 301         writeb(IOAT_CHANCMD_RESET,
 302                ioat_chan->reg_base
 303                + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
 304         spin_unlock_bh(&ioat_chan->desc_lock);
 305
 306         /* schedule the 2nd half instead of sleeping a long time */
 307         schedule_delayed_work(&ioat_chan->work, RESET_DELAY);
 308 }
 309
 310 /**
 311  * ioat_dma_chan_watchdog - watch for stuck channels
 312  */
 313 static void ioat_dma_chan_watchdog(struct work_struct *work)
 314 {
 315         struct ioatdma_device *device =
 316                 container_of(work, struct ioatdma_device, work.work);
 317         struct ioat_dma_chan *ioat_chan;
 318         int i;
 319
 320         union {
 321                 u64 full;
 322                 struct {
 323                         u32 low;
 324                         u32 high;
 325                 };
 326         } completion_hw;
 327         unsigned long compl_desc_addr_hw;
 328
 329         for (i = 0; i < device->common.chancnt; i++) {
 330                 ioat_chan = ioat_chan_by_index(device, i);
 331
 332                 if (ioat_chan->device->version == IOAT_VER_1_2
 333                         /* have we started processing anything yet */
 334                     && ioat_chan->last_completion
 335                         /* have we completed any since last watchdog cycle? */
 336                     && (ioat_chan->last_completion ==
 337                                 ioat_chan->watchdog_completion)
 338                         /* has TCP stuck on one cookie since last watchdog? */
 339                     && (ioat_chan->watchdog_tcp_cookie ==
 340                                 ioat_chan->watchdog_last_tcp_cookie)
 341                     && (ioat_chan->watchdog_tcp_cookie !=
 342                                 ioat_chan->completed_cookie)
 343                         /* is there something in the chain to be processed? */
 344                         /* CB1 chain always has at least the last one processed */
 345                     && (ioat_chan->used_desc.prev != ioat_chan->used_desc.next)
 346                     && ioat_chan->pending == 0) {
 347
 348                         /*
 349                          * check CHANSTS register for completed
 350                          * descriptor address.
 351                          * if it is different than completion writeback,
 352                          * it is not zero
 353                          * and it has changed since the last watchdog
 354                          *     we can assume that channel
 355                          *     is still working correctly
 356                          *     and the problem is in completion writeback.
 357                          *     update completion writeback
 358                          *     with actual CHANSTS value
 359                          * else
 360                          *     try resetting the channel
 361                          */
 362
 363                         completion_hw.low = readl(ioat_chan->reg_base +
 364                                 IOAT_CHANSTS_OFFSET_LOW(ioat_chan->device->version));
 365                         completion_hw.high = readl(ioat_chan->reg_base +
 366                                 IOAT_CHANSTS_OFFSET_HIGH(ioat_chan->device->version));
 367 #if (BITS_PER_LONG == 64)
 368                         compl_desc_addr_hw =
 369                                 completion_hw.full
 370                                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
 371 #else
 372                         compl_desc_addr_hw =
 373                                 completion_hw.low & IOAT_LOW_COMPLETION_MASK;
 374 #endif
 375
 376                         if ((compl_desc_addr_hw != 0)
 377                            && (compl_desc_addr_hw != ioat_chan->watchdog_completion)
 378                            && (compl_desc_addr_hw != ioat_chan->last_compl_desc_addr_hw)) {
 379                                 ioat_chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
 380                                 ioat_chan->completion_virt->low = completion_hw.low;
 381                                 ioat_chan->completion_virt->high = completion_hw.high;
 382                         } else {
 383                                 ioat_dma_reset_channel(ioat_chan);
 384                                 ioat_chan->watchdog_completion = 0;
 385                                 ioat_chan->last_compl_desc_addr_hw = 0;
 386                         }
 387
 388                 /*
 389                  * for version 2.0 if there are descriptors yet to be processed
 390                  * and the last completed hasn't changed since the last watchdog
 391                  *      if they haven't hit the pending level
 392                  *          issue the pending to push them through
 393                  *      else
 394                  *          try resetting the channel
 395                  */
 396                 } else if (ioat_chan->device->version == IOAT_VER_2_0
 397                     && ioat_chan->used_desc.prev
 398                     && ioat_chan->last_completion
 399                     && ioat_chan->last_completion == ioat_chan->watchdog_completion) {
 400
 401                         if (ioat_chan->pending < ioat_pending_level)
 402                                 ioat2_dma_memcpy_issue_pending(&ioat_chan->common);
 403                         else {
 404                                 ioat_dma_reset_channel(ioat_chan);
 405                                 ioat_chan->watchdog_completion = 0;
 406                         }
 407                 } else {
 408                         ioat_chan->last_compl_desc_addr_hw = 0;
 409                         ioat_chan->watchdog_completion
 410                                         = ioat_chan->last_completion;
 411                 }
 412
 413                 ioat_chan->watchdog_last_tcp_cookie =
 414                         ioat_chan->watchdog_tcp_cookie;
 415         }
 416
 417         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
 418 }
 419
 420 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
 421 {
 422         struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
 423         struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
 424         struct ioat_desc_sw *prev, *new;
 425         struct ioat_dma_descriptor *hw;
 426         dma_cookie_t cookie;
 427         LIST_HEAD(new_chain);
 428         u32 copy;
 429         size_t len;
 430         dma_addr_t src, dst;
 431         unsigned long orig_flags;
 432         unsigned int desc_count = 0;
 433
 434         /* src and dest and len are stored in the initial descriptor */
 435         len = first->len;
 436         src = first->src;
 437         dst = first->dst;
 438         orig_flags = first->txd.flags;
 439         new = first;
 440
 441         spin_lock_bh(&ioat_chan->desc_lock);
 442         prev = to_ioat_desc(ioat_chan->used_desc.prev);
 443         prefetch(prev->hw);
 444         do {
 445                 copy = min_t(size_t, len, ioat_chan->xfercap);
 446
 447                 async_tx_ack(&new->txd);
 448
 449                 hw = new->hw;
 450                 hw->size = copy;
 451                 hw->ctl = 0;
 452                 hw->src_addr = src;
 453                 hw->dst_addr = dst;
 454                 hw->next = 0;
 455
 456                 /* chain together the physical address list for the HW */
 457                 wmb();
 458                 prev->hw->next = (u64) new->txd.phys;
 459
 460                 len -= copy;
 461                 dst += copy;
 462                 src += copy;
 463
 464                 list_add_tail(&new->node, &new_chain);
 465                 desc_count++;
 466                 prev = new;
 467         } while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));
 468
 469         if (!new) {
 470                 dev_err(to_dev(ioat_chan), "tx submit failed\n");
 471                 spin_unlock_bh(&ioat_chan->desc_lock);
 472                 return -ENOMEM;
 473         }
 474
 475         hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
 476         if (first->txd.callback) {
 477                 hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
 478                 if (first != new) {
 479                         /* move callback into to last desc */
 480                         new->txd.callback = first->txd.callback;
 481                         new->txd.callback_param
 482                                         = first->txd.callback_param;
 483                         first->txd.callback = NULL;
 484                         first->txd.callback_param = NULL;
 485                 }
 486         }
 487
 488         new->tx_cnt = desc_count;
 489         new->txd.flags = orig_flags; /* client is in control of this ack */
 490
 491         /* store the original values for use in later cleanup */
 492         if (new != first) {
 493                 new->src = first->src;
 494                 new->dst = first->dst;
 495                 new->len = first->len;
 496         }
 497
 498         /* cookie incr and addition to used_list must be atomic */
 499         cookie = ioat_chan->common.cookie;
 500         cookie++;
 501         if (cookie < 0)
 502                 cookie = 1;
 503         ioat_chan->common.cookie = new->txd.cookie = cookie;
 504
 505         /* write address into NextDescriptor field of last desc in chain */
 506         to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
 507                                                         first->txd.phys;
 508         list_splice_tail(&new_chain, &ioat_chan->used_desc);
 509
 510         ioat_chan->dmacount += desc_count;
 511         ioat_chan->pending += desc_count;
 512         if (ioat_chan->pending >= ioat_pending_level)
 513                 __ioat1_dma_memcpy_issue_pending(ioat_chan);
 514         spin_unlock_bh(&ioat_chan->desc_lock);
 515
 516         return cookie;
 517 }
 518
 519 static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
 520 {
 521         struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
 522         struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
 523         struct ioat_desc_sw *new;
 524         struct ioat_dma_descriptor *hw;
 525         dma_cookie_t cookie;
 526         u32 copy;
 527         size_t len;
 528         dma_addr_t src, dst;
 529         unsigned long orig_flags;
 530         unsigned int desc_count = 0;
 531
 532         /* src and dest and len are stored in the initial descriptor */
 533         len = first->len;
 534         src = first->src;
 535         dst = first->dst;
 536         orig_flags = first->txd.flags;
 537         new = first;
 538
 539         /*
 540          * ioat_chan->desc_lock is still in force in version 2 path
 541          * it gets unlocked at end of this function
 542          */
 543         do {
 544                 copy = min_t(size_t, len, ioat_chan->xfercap);
 545
 546                 async_tx_ack(&new->txd);
 547
 548                 hw = new->hw;
 549                 hw->size = copy;
 550                 hw->ctl = 0;
 551                 hw->src_addr = src;
 552                 hw->dst_addr = dst;
 553
 554                 len -= copy;
 555                 dst += copy;
 556                 src += copy;
 557                 desc_count++;
 558         } while (len && (new = ioat2_dma_get_next_descriptor(ioat_chan)));
 559
 560         if (!new) {
 561                 dev_err(to_dev(ioat_chan), "tx submit failed\n");
 562                 spin_unlock_bh(&ioat_chan->desc_lock);
 563                 return -ENOMEM;
 564         }
 565
 566         hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
 567         if (first->txd.callback) {
 568                 hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
 569                 if (first != new) {
 570                         /* move callback into to last desc */
 571                         new->txd.callback = first->txd.callback;
 572                         new->txd.callback_param
 573                                         = first->txd.callback_param;
 574                         first->txd.callback = NULL;
 575                         first->txd.callback_param = NULL;
 576                 }
 577         }
 578
 579         new->tx_cnt = desc_count;
 580         new->txd.flags = orig_flags; /* client is in control of this ack */
 581
 582         /* store the original values for use in later cleanup */
 583         if (new != first) {
 584                 new->src = first->src;
 585                 new->dst = first->dst;
 586                 new->len = first->len;
 587         }
 588
 589         /* cookie incr and addition to used_list must be atomic */
 590         cookie = ioat_chan->common.cookie;
 591         cookie++;
 592         if (cookie < 0)
 593                 cookie = 1;
 594         ioat_chan->common.cookie = new->txd.cookie = cookie;
 595
 596         ioat_chan->dmacount += desc_count;
 597         ioat_chan->pending += desc_count;
 598         if (ioat_chan->pending >= ioat_pending_level)
 599                 __ioat2_dma_memcpy_issue_pending(ioat_chan);
 600         spin_unlock_bh(&ioat_chan->desc_lock);
 601
 602         return cookie;
 603 }
 604
 605 /**
 606  * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
 607  * @ioat_chan: the channel supplying the memory pool for the descriptors
 608  * @flags: allocation flags
 609  */
 610 static struct ioat_desc_sw *
 611 ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat_chan, gfp_t flags)
 612 {
 613         struct ioat_dma_descriptor *desc;
 614         struct ioat_desc_sw *desc_sw;
 615         struct ioatdma_device *ioatdma_device;
 616         dma_addr_t phys;
 617
 618         ioatdma_device = to_ioatdma_device(ioat_chan->common.device);
 619         desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
 620         if (unlikely(!desc))
 621                 return NULL;
 622
 623         desc_sw = kzalloc(sizeof(*desc_sw), flags);
 624         if (unlikely(!desc_sw)) {
 625                 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
 626                 return NULL;
 627         }
 628
 629         memset(desc, 0, sizeof(*desc));
 630         dma_async_tx_descriptor_init(&desc_sw->txd, &ioat_chan->common);
 631         switch (ioat_chan->device->version) {
 632         case IOAT_VER_1_2:
 633                 desc_sw->txd.tx_submit = ioat1_tx_submit;
 634                 break;
 635         case IOAT_VER_2_0:
 636         case IOAT_VER_3_0:
 637                 desc_sw->txd.tx_submit = ioat2_tx_submit;
 638                 break;
 639         }
 640
 641         desc_sw->hw = desc;
 642         desc_sw->txd.phys = phys;
 643
 644         return desc_sw;
 645 }
 646
 647 static int ioat_initial_desc_count = 256;
 648 module_param(ioat_initial_desc_count, int, 0644);
 649 MODULE_PARM_DESC(ioat_initial_desc_count,
 650                  "initial descriptors per channel (default: 256)");
 651
 652 /**
 653  * ioat2_dma_massage_chan_desc - link the descriptors into a circle
 654  * @ioat_chan: the channel to be massaged
 655  */
 656 static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat_chan)
 657 {
 658         struct ioat_desc_sw *desc, *_desc;
 659
 660         /* setup used_desc */
 661         ioat_chan->used_desc.next = ioat_chan->free_desc.next;
 662         ioat_chan->used_desc.prev = NULL;
 663
 664         /* pull free_desc out of the circle so that every node is a hw
 665          * descriptor, but leave it pointing to the list
 666          */
 667         ioat_chan->free_desc.prev->next = ioat_chan->free_desc.next;
 668         ioat_chan->free_desc.next->prev = ioat_chan->free_desc.prev;
 669
 670         /* circle link the hw descriptors */
 671         desc = to_ioat_desc(ioat_chan->free_desc.next);
 672         desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
 673         list_for_each_entry_safe(desc, _desc, ioat_chan->free_desc.next, node) {
 674                 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
 675         }
 676 }
 677
 678 /**
 679  * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors
 680  * @chan: the channel to be filled out
 681  */
 682 static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
 683 {
 684         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 685         struct ioat_desc_sw *desc;
 686         u16 chanctrl;
 687         u32 chanerr;
 688         int i;
 689         LIST_HEAD(tmp_list);
 690
 691         /* have we already been set up? */
 692         if (!list_empty(&ioat_chan->free_desc))
 693                 return ioat_chan->desccount;
 694
 695         /* Setup register to interrupt and write completion status on error */
 696         chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
 697                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 698                 IOAT_CHANCTRL_ERR_COMPLETION_EN;
 699         writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
 700
 701         chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 702         if (chanerr) {
 703                 dev_err(to_dev(ioat_chan), "CHANERR = %x, clearing\n", chanerr);
 704                 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 705         }
 706
 707         /* Allocate descriptors */
 708         for (i = 0; i < ioat_initial_desc_count; i++) {
 709                 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
 710                 if (!desc) {
 711                         dev_err(to_dev(ioat_chan),
 712                                 "Only %d initial descriptors\n", i);
 713                         break;
 714                 }
 715                 list_add_tail(&desc->node, &tmp_list);
 716         }
 717         spin_lock_bh(&ioat_chan->desc_lock);
 718         ioat_chan->desccount = i;
 719         list_splice(&tmp_list, &ioat_chan->free_desc);
 720         if (ioat_chan->device->version != IOAT_VER_1_2)
 721                 ioat2_dma_massage_chan_desc(ioat_chan);
 722         spin_unlock_bh(&ioat_chan->desc_lock);
 723
 724         /* allocate a completion writeback area */
 725         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
 726         ioat_chan->completion_virt =
 727                 pci_pool_alloc(ioat_chan->device->completion_pool,
 728                                GFP_KERNEL,
 729                                &ioat_chan->completion_addr);
 730         memset(ioat_chan->completion_virt, 0,
 731                sizeof(*ioat_chan->completion_virt));
 732         writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
 733                ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
 734         writel(((u64) ioat_chan->completion_addr) >> 32,
 735                ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
 736
 737         tasklet_enable(&ioat_chan->cleanup_task);
 738         ioat_dma_start_null_desc(ioat_chan);  /* give chain to dma device */
 739         return ioat_chan->desccount;
 740 }
 741
 742 /**
 743  * ioat_dma_free_chan_resources - release all the descriptors
 744  * @chan: the channel to be cleaned
 745  */
 746 static void ioat_dma_free_chan_resources(struct dma_chan *chan)
 747 {
 748         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 749         struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device);
 750         struct ioat_desc_sw *desc, *_desc;
 751         int in_use_descs = 0;
 752
 753         /* Before freeing channel resources first check
 754          * if they have been previously allocated for this channel.
 755          */
 756         if (ioat_chan->desccount == 0)
 757                 return;
 758
 759         tasklet_disable(&ioat_chan->cleanup_task);
 760         ioat_dma_memcpy_cleanup(ioat_chan);
 761
 762         /* Delay 100ms after reset to allow internal DMA logic to quiesce
 763          * before removing DMA descriptor resources.
 764          */
 765         writeb(IOAT_CHANCMD_RESET,
 766                ioat_chan->reg_base
 767                         + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
 768         mdelay(100);
 769
 770         spin_lock_bh(&ioat_chan->desc_lock);
 771         switch (ioat_chan->device->version) {
 772         case IOAT_VER_1_2:
 773                 list_for_each_entry_safe(desc, _desc,
 774                                          &ioat_chan->used_desc, node) {
 775                         in_use_descs++;
 776                         list_del(&desc->node);
 777                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 778                                       desc->txd.phys);
 779                         kfree(desc);
 780                 }
 781                 list_for_each_entry_safe(desc, _desc,
 782                                          &ioat_chan->free_desc, node) {
 783                         list_del(&desc->node);
 784                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 785                                       desc->txd.phys);
 786                         kfree(desc);
 787                 }
 788                 break;
 789         case IOAT_VER_2_0:
 790         case IOAT_VER_3_0:
 791                 list_for_each_entry_safe(desc, _desc,
 792                                          ioat_chan->free_desc.next, node) {
 793                         list_del(&desc->node);
 794                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 795                                       desc->txd.phys);
 796                         kfree(desc);
 797                 }
 798                 desc = to_ioat_desc(ioat_chan->free_desc.next);
 799                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 800                               desc->txd.phys);
 801                 kfree(desc);
 802                 INIT_LIST_HEAD(&ioat_chan->free_desc);
 803                 INIT_LIST_HEAD(&ioat_chan->used_desc);
 804                 break;
 805         }
 806         spin_unlock_bh(&ioat_chan->desc_lock);
 807
 808         pci_pool_free(ioatdma_device->completion_pool,
 809                       ioat_chan->completion_virt,
 810                       ioat_chan->completion_addr);
 811
 812         /* one is ok since we left it on there on purpose */
 813         if (in_use_descs > 1)
 814                 dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
 815                         in_use_descs - 1);
 816
 817         ioat_chan->last_completion = ioat_chan->completion_addr = 0;
 818         ioat_chan->pending = 0;
 819         ioat_chan->dmacount = 0;
 820         ioat_chan->desccount = 0;
 821         ioat_chan->watchdog_completion = 0;
 822         ioat_chan->last_compl_desc_addr_hw = 0;
 823         ioat_chan->watchdog_tcp_cookie =
 824                 ioat_chan->watchdog_last_tcp_cookie = 0;
 825 }
 826
 827 /**
 828  * ioat_dma_get_next_descriptor - return the next available descriptor
 829  * @ioat_chan: IOAT DMA channel handle
 830  *
 831  * Gets the next descriptor from the chain, and must be called with the
 832  * channel's desc_lock held.  Allocates more descriptors if the channel
 833  * has run out.
 834  */
 835 static struct ioat_desc_sw *
 836 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
 837 {
 838         struct ioat_desc_sw *new;
 839
 840         if (!list_empty(&ioat_chan->free_desc)) {
 841                 new = to_ioat_desc(ioat_chan->free_desc.next);
 842                 list_del(&new->node);
 843         } else {
 844                 /* try to get another desc */
 845                 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
 846                 if (!new) {
 847                         dev_err(to_dev(ioat_chan), "alloc failed\n");
 848                         return NULL;
 849                 }
 850         }
 851
 852         prefetch(new->hw);
 853         return new;
 854 }
 855
 856 static struct ioat_desc_sw *
 857 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
 858 {
 859         struct ioat_desc_sw *new;
 860
 861         /*
 862          * used.prev points to where to start processing
 863          * used.next points to next free descriptor
 864          * if used.prev == NULL, there are none waiting to be processed
 865          * if used.next == used.prev.prev, there is only one free descriptor,
 866          *      and we need to use it to as a noop descriptor before
 867          *      linking in a new set of descriptors, since the device
 868          *      has probably already read the pointer to it
 869          */
 870         if (ioat_chan->used_desc.prev &&
 871             ioat_chan->used_desc.next == ioat_chan->used_desc.prev->prev) {
 872
 873                 struct ioat_desc_sw *desc;
 874                 struct ioat_desc_sw *noop_desc;
 875                 int i;
 876
 877                 /* set up the noop descriptor */
 878                 noop_desc = to_ioat_desc(ioat_chan->used_desc.next);
 879                 /* set size to non-zero value (channel returns error when size is 0) */
 880                 noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
 881                 noop_desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
 882                 noop_desc->hw->src_addr = 0;
 883                 noop_desc->hw->dst_addr = 0;
 884
 885                 ioat_chan->used_desc.next = ioat_chan->used_desc.next->next;
 886                 ioat_chan->pending++;
 887                 ioat_chan->dmacount++;
 888
 889                 /* try to get a few more descriptors */
 890                 for (i = 16; i; i--) {
 891                         desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
 892                         if (!desc) {
 893                                 dev_err(to_dev(ioat_chan), "alloc failed\n");
 894                                 break;
 895                         }
 896                         list_add_tail(&desc->node, ioat_chan->used_desc.next);
 897
 898                         desc->hw->next
 899                                 = to_ioat_desc(desc->node.next)->txd.phys;
 900                         to_ioat_desc(desc->node.prev)->hw->next
 901                                 = desc->txd.phys;
 902                         ioat_chan->desccount++;
 903                 }
 904
 905                 ioat_chan->used_desc.next = noop_desc->node.next;
 906         }
 907         new = to_ioat_desc(ioat_chan->used_desc.next);
 908         prefetch(new);
 909         ioat_chan->used_desc.next = new->node.next;
 910
 911         if (ioat_chan->used_desc.prev == NULL)
 912                 ioat_chan->used_desc.prev = &new->node;
 913
 914         prefetch(new->hw);
 915         return new;
 916 }
 917
 918 static struct ioat_desc_sw *
 919 ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
 920 {
 921         if (!ioat_chan)
 922                 return NULL;
 923
 924         switch (ioat_chan->device->version) {
 925         case IOAT_VER_1_2:
 926                 return ioat1_dma_get_next_descriptor(ioat_chan);
 927         case IOAT_VER_2_0:
 928         case IOAT_VER_3_0:
 929                 return ioat2_dma_get_next_descriptor(ioat_chan);
 930         }
 931         return NULL;
 932 }
 933
 934 static struct dma_async_tx_descriptor *
 935 ioat1_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 936                       dma_addr_t dma_src, size_t len, unsigned long flags)
 937 {
 938         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 939         struct ioat_desc_sw *new;
 940
 941         spin_lock_bh(&ioat_chan->desc_lock);
 942         new = ioat_dma_get_next_descriptor(ioat_chan);
 943         spin_unlock_bh(&ioat_chan->desc_lock);
 944
 945         if (new) {
 946                 new->len = len;
 947                 new->dst = dma_dest;
 948                 new->src = dma_src;
 949                 new->txd.flags = flags;
 950                 return &new->txd;
 951         } else {
 952                 dev_err(to_dev(ioat_chan),
 953                         "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
 954                         chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
 955                 return NULL;
 956         }
 957 }
 958
 959 static struct dma_async_tx_descriptor *
 960 ioat2_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 961                       dma_addr_t dma_src, size_t len, unsigned long flags)
 962 {
 963         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 964         struct ioat_desc_sw *new;
 965
 966         spin_lock_bh(&ioat_chan->desc_lock);
 967         new = ioat2_dma_get_next_descriptor(ioat_chan);
 968
 969         /*
 970          * leave ioat_chan->desc_lock set in ioat 2 path
 971          * it will get unlocked at end of tx_submit
 972          */
 973
 974         if (new) {
 975                 new->len = len;
 976                 new->dst = dma_dest;
 977                 new->src = dma_src;
 978                 new->txd.flags = flags;
 979                 return &new->txd;
 980         } else {
 981                 spin_unlock_bh(&ioat_chan->desc_lock);
 982                 dev_err(to_dev(ioat_chan),
 983                         "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
 984                         chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
 985                 return NULL;
 986         }
 987 }
 988
 989 static void ioat_dma_cleanup_tasklet(unsigned long data)
 990 {
 991         struct ioat_dma_chan *chan = (void *)data;
 992         ioat_dma_memcpy_cleanup(chan);
 993         writew(IOAT_CHANCTRL_INT_DISABLE,
 994                chan->reg_base + IOAT_CHANCTRL_OFFSET);
 995 }
 996
 997 static void
 998 ioat_dma_unmap(struct ioat_dma_chan *ioat_chan, struct ioat_desc_sw *desc)
 999 {
1000         if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1001                 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
1002                         pci_unmap_single(ioat_chan->device->pdev,
1003                                          pci_unmap_addr(desc, dst),
1004                                          pci_unmap_len(desc, len),
1005                                          PCI_DMA_FROMDEVICE);
1006                 else
1007                         pci_unmap_page(ioat_chan->device->pdev,
1008                                        pci_unmap_addr(desc, dst),
1009                                        pci_unmap_len(desc, len),
1010                                        PCI_DMA_FROMDEVICE);
1011         }
1012
1013         if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1014                 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1015                         pci_unmap_single(ioat_chan->device->pdev,
1016                                          pci_unmap_addr(desc, src),
1017                                          pci_unmap_len(desc, len),
1018                                          PCI_DMA_TODEVICE);
1019                 else
1020                         pci_unmap_page(ioat_chan->device->pdev,
1021                                        pci_unmap_addr(desc, src),
1022                                        pci_unmap_len(desc, len),
1023                                        PCI_DMA_TODEVICE);
1024         }
1025 }
1026
1027 /**
1028  * ioat_dma_memcpy_cleanup - cleanup up finished descriptors
1029  * @chan: ioat channel to be cleaned up
1030  */
1031 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan)
1032 {
1033         unsigned long phys_complete;
1034         struct ioat_desc_sw *desc, *_desc;
1035         dma_cookie_t cookie = 0;
1036         unsigned long desc_phys;
1037         struct ioat_desc_sw *latest_desc;
1038         struct dma_async_tx_descriptor *tx;
1039
1040         prefetch(ioat_chan->completion_virt);
1041
1042         if (!spin_trylock_bh(&ioat_chan->cleanup_lock))
1043                 return;
1044
1045         /* The completion writeback can happen at any time,
1046            so reads by the driver need to be atomic operations
1047            The descriptor physical addresses are limited to 32-bits
1048            when the CPU can only do a 32-bit mov */
1049
1050 #if (BITS_PER_LONG == 64)
1051         phys_complete =
1052                 ioat_chan->completion_virt->full
1053                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1054 #else
1055         phys_complete =
1056                 ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
1057 #endif
1058
1059         if ((ioat_chan->completion_virt->full
1060                 & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
1061                                 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
1062                 dev_err(to_dev(ioat_chan), "Channel halted, chanerr = %x\n",
1063                         readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET));
1064
1065                 /* TODO do something to salvage the situation */
1066         }
1067
1068         if (phys_complete == ioat_chan->last_completion) {
1069                 spin_unlock_bh(&ioat_chan->cleanup_lock);
1070                 /*
1071                  * perhaps we're stuck so hard that the watchdog can't go off?
1072                  * try to catch it after 2 seconds
1073                  */
1074                 if (ioat_chan->device->version != IOAT_VER_3_0) {
1075                         if (time_after(jiffies,
1076                                        ioat_chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
1077                                 ioat_dma_chan_watchdog(&(ioat_chan->device->work.work));
1078                                 ioat_chan->last_completion_time = jiffies;
1079                         }
1080                 }
1081                 return;
1082         }
1083         ioat_chan->last_completion_time = jiffies;
1084
1085         cookie = 0;
1086         if (!spin_trylock_bh(&ioat_chan->desc_lock)) {
1087                 spin_unlock_bh(&ioat_chan->cleanup_lock);
1088                 return;
1089         }
1090
1091         switch (ioat_chan->device->version) {
1092         case IOAT_VER_1_2:
1093                 list_for_each_entry_safe(desc, _desc,
1094                                          &ioat_chan->used_desc, node) {
1095                         tx = &desc->txd;
1096                         /*
1097                          * Incoming DMA requests may use multiple descriptors,
1098                          * due to exceeding xfercap, perhaps. If so, only the
1099                          * last one will have a cookie, and require unmapping.
1100                          */
1101                         if (tx->cookie) {
1102                                 cookie = tx->cookie;
1103                                 ioat_dma_unmap(ioat_chan, desc);
1104                                 if (tx->callback) {
1105                                         tx->callback(tx->callback_param);
1106                                         tx->callback = NULL;
1107                                 }
1108                         }
1109
1110                         if (tx->phys != phys_complete) {
1111                                 /*
1112                                  * a completed entry, but not the last, so clean
1113                                  * up if the client is done with the descriptor
1114                                  */
1115                                 if (async_tx_test_ack(tx)) {
1116                                         list_move_tail(&desc->node,
1117                                                        &ioat_chan->free_desc);
1118                                 } else
1119                                         tx->cookie = 0;
1120                         } else {
1121                                 /*
1122                                  * last used desc. Do not remove, so we can
1123                                  * append from it, but don't look at it next
1124                                  * time, either
1125                                  */
1126                                 tx->cookie = 0;
1127
1128                                 /* TODO check status bits? */
1129                                 break;
1130                         }
1131                 }
1132                 break;
1133         case IOAT_VER_2_0:
1134         case IOAT_VER_3_0:
1135                 /* has some other thread has already cleaned up? */
1136                 if (ioat_chan->used_desc.prev == NULL)
1137                         break;
1138
1139                 /* work backwards to find latest finished desc */
1140                 desc = to_ioat_desc(ioat_chan->used_desc.next);
1141                 tx = &desc->txd;
1142                 latest_desc = NULL;
1143                 do {
1144                         desc = to_ioat_desc(desc->node.prev);
1145                         desc_phys = (unsigned long)tx->phys
1146                                        & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1147                         if (desc_phys == phys_complete) {
1148                                 latest_desc = desc;
1149                                 break;
1150                         }
1151                 } while (&desc->node != ioat_chan->used_desc.prev);
1152
1153                 if (latest_desc != NULL) {
1154                         /* work forwards to clear finished descriptors */
1155                         for (desc = to_ioat_desc(ioat_chan->used_desc.prev);
1156                              &desc->node != latest_desc->node.next &&
1157                              &desc->node != ioat_chan->used_desc.next;
1158                              desc = to_ioat_desc(desc->node.next)) {
1159                                 if (tx->cookie) {
1160                                         cookie = tx->cookie;
1161                                         tx->cookie = 0;
1162                                         ioat_dma_unmap(ioat_chan, desc);
1163                                         if (tx->callback) {
1164                                                 tx->callback(tx->callback_param);
1165                                                 tx->callback = NULL;
1166                                         }
1167                                 }
1168                         }
1169
1170                         /* move used.prev up beyond those that are finished */
1171                         if (&desc->node == ioat_chan->used_desc.next)
1172                                 ioat_chan->used_desc.prev = NULL;
1173                         else
1174                                 ioat_chan->used_desc.prev = &desc->node;
1175                 }
1176                 break;
1177         }
1178
1179         spin_unlock_bh(&ioat_chan->desc_lock);
1180
1181         ioat_chan->last_completion = phys_complete;
1182         if (cookie != 0)
1183                 ioat_chan->completed_cookie = cookie;
1184
1185         spin_unlock_bh(&ioat_chan->cleanup_lock);
1186 }
1187
1188 /**
1189  * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
1190  * @chan: IOAT DMA channel handle
1191  * @cookie: DMA transaction identifier
1192  * @done: if not %NULL, updated with last completed transaction
1193  * @used: if not %NULL, updated with last used transaction
1194  */
1195 static enum dma_status
1196 ioat_dma_is_complete(struct dma_chan *chan, dma_cookie_t cookie,
1197                      dma_cookie_t *done, dma_cookie_t *used)
1198 {
1199         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
1200         dma_cookie_t last_used;
1201         dma_cookie_t last_complete;
1202         enum dma_status ret;
1203
1204         last_used = chan->cookie;
1205         last_complete = ioat_chan->completed_cookie;
1206         ioat_chan->watchdog_tcp_cookie = cookie;
1207
1208         if (done)
1209                 *done = last_complete;
1210         if (used)
1211                 *used = last_used;
1212
1213         ret = dma_async_is_complete(cookie, last_complete, last_used);
1214         if (ret == DMA_SUCCESS)
1215                 return ret;
1216
1217         ioat_dma_memcpy_cleanup(ioat_chan);
1218
1219         last_used = chan->cookie;
1220         last_complete = ioat_chan->completed_cookie;
1221
1222         if (done)
1223                 *done = last_complete;
1224         if (used)
1225                 *used = last_used;
1226
1227         return dma_async_is_complete(cookie, last_complete, last_used);
1228 }
1229
1230 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan)
1231 {
1232         struct ioat_desc_sw *desc;
1233
1234         spin_lock_bh(&ioat_chan->desc_lock);
1235
1236         desc = ioat_dma_get_next_descriptor(ioat_chan);
1237
1238         if (!desc) {
1239                 dev_err(to_dev(ioat_chan),
1240                         "Unable to start null desc - get next desc failed\n");
1241                 spin_unlock_bh(&ioat_chan->desc_lock);
1242                 return;
1243         }
1244
1245         desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL
1246                                 | IOAT_DMA_DESCRIPTOR_CTL_INT_GN
1247                                 | IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
1248         /* set size to non-zero value (channel returns error when size is 0) */
1249         desc->hw->size = NULL_DESC_BUFFER_SIZE;
1250         desc->hw->src_addr = 0;
1251         desc->hw->dst_addr = 0;
1252         async_tx_ack(&desc->txd);
1253         switch (ioat_chan->device->version) {
1254         case IOAT_VER_1_2:
1255                 desc->hw->next = 0;
1256                 list_add_tail(&desc->node, &ioat_chan->used_desc);
1257
1258                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1259                        ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
1260                 writel(((u64) desc->txd.phys) >> 32,
1261                        ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
1262
1263                 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
1264                         + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
1265                 break;
1266         case IOAT_VER_2_0:
1267         case IOAT_VER_3_0:
1268                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1269                        ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
1270                 writel(((u64) desc->txd.phys) >> 32,
1271                        ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
1272
1273                 ioat_chan->dmacount++;
1274                 __ioat2_dma_memcpy_issue_pending(ioat_chan);
1275                 break;
1276         }
1277         spin_unlock_bh(&ioat_chan->desc_lock);
1278 }
1279
1280 /*
1281  * Perform a IOAT transaction to verify the HW works.
1282  */
1283 #define IOAT_TEST_SIZE 2000
1284
1285 static void ioat_dma_test_callback(void *dma_async_param)
1286 {
1287         struct completion *cmp = dma_async_param;
1288
1289         complete(cmp);
1290 }
1291
1292 /**
1293  * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
1294  * @device: device to be tested
1295  */
1296 static int ioat_dma_self_test(struct ioatdma_device *device)
1297 {
1298         int i;
1299         u8 *src;
1300         u8 *dest;
1301         struct dma_device *dma = &device->common;
1302         struct device *dev = &device->pdev->dev;
1303         struct dma_chan *dma_chan;
1304         struct dma_async_tx_descriptor *tx;
1305         dma_addr_t dma_dest, dma_src;
1306         dma_cookie_t cookie;
1307         int err = 0;
1308         struct completion cmp;
1309         unsigned long tmo;
1310         unsigned long flags;
1311
1312         src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1313         if (!src)
1314                 return -ENOMEM;
1315         dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1316         if (!dest) {
1317                 kfree(src);
1318                 return -ENOMEM;
1319         }
1320
1321         /* Fill in src buffer */
1322         for (i = 0; i < IOAT_TEST_SIZE; i++)
1323                 src[i] = (u8)i;
1324
1325         /* Start copy, using first DMA channel */
1326         dma_chan = container_of(dma->channels.next, struct dma_chan,
1327                                 device_node);
1328         if (dma->device_alloc_chan_resources(dma_chan) < 1) {
1329                 dev_err(dev, "selftest cannot allocate chan resource\n");
1330                 err = -ENODEV;
1331                 goto out;
1332         }
1333
1334         dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
1335         dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
1336         flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE;
1337         tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
1338                                                    IOAT_TEST_SIZE, flags);
1339         if (!tx) {
1340                 dev_err(dev, "Self-test prep failed, disabling\n");
1341                 err = -ENODEV;
1342                 goto free_resources;
1343         }
1344
1345         async_tx_ack(tx);
1346         init_completion(&cmp);
1347         tx->callback = ioat_dma_test_callback;
1348         tx->callback_param = &cmp;
1349         cookie = tx->tx_submit(tx);
1350         if (cookie < 0) {
1351                 dev_err(dev, "Self-test setup failed, disabling\n");
1352                 err = -ENODEV;
1353                 goto free_resources;
1354         }
1355         dma->device_issue_pending(dma_chan);
1356
1357         tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1358
1359         if (tmo == 0 ||
1360             dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
1361                                         != DMA_SUCCESS) {
1362                 dev_err(dev, "Self-test copy timed out, disabling\n");
1363                 err = -ENODEV;
1364                 goto free_resources;
1365         }
1366         if (memcmp(src, dest, IOAT_TEST_SIZE)) {
1367                 dev_err(dev, "Self-test copy failed compare, disabling\n");
1368                 err = -ENODEV;
1369                 goto free_resources;
1370         }
1371
1372 free_resources:
1373         dma->device_free_chan_resources(dma_chan);
1374 out:
1375         kfree(src);
1376         kfree(dest);
1377         return err;
1378 }
1379
1380 static char ioat_interrupt_style[32] = "msix";
1381 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
1382                     sizeof(ioat_interrupt_style), 0644);
1383 MODULE_PARM_DESC(ioat_interrupt_style,
1384                  "set ioat interrupt style: msix (default), "
1385                  "msix-single-vector, msi, intx)");
1386
1387 /**
1388  * ioat_dma_setup_interrupts - setup interrupt handler
1389  * @device: ioat device
1390  */
1391 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
1392 {
1393         struct ioat_dma_chan *ioat_chan;
1394         struct pci_dev *pdev = device->pdev;
1395         struct device *dev = &pdev->dev;
1396         struct msix_entry *msix;
1397         int i, j, msixcnt;
1398         int err = -EINVAL;
1399         u8 intrctrl = 0;
1400
1401         if (!strcmp(ioat_interrupt_style, "msix"))
1402                 goto msix;
1403         if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
1404                 goto msix_single_vector;
1405         if (!strcmp(ioat_interrupt_style, "msi"))
1406                 goto msi;
1407         if (!strcmp(ioat_interrupt_style, "intx"))
1408                 goto intx;
1409         dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
1410         goto err_no_irq;
1411
1412 msix:
1413         /* The number of MSI-X vectors should equal the number of channels */
1414         msixcnt = device->common.chancnt;
1415         for (i = 0; i < msixcnt; i++)
1416                 device->msix_entries[i].entry = i;
1417
1418         err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
1419         if (err < 0)
1420                 goto msi;
1421         if (err > 0)
1422                 goto msix_single_vector;
1423
1424         for (i = 0; i < msixcnt; i++) {
1425                 msix = &device->msix_entries[i];
1426                 ioat_chan = ioat_chan_by_index(device, i);
1427                 err = devm_request_irq(dev, msix->vector,
1428                                        ioat_dma_do_interrupt_msix, 0,
1429                                        "ioat-msix", ioat_chan);
1430                 if (err) {
1431                         for (j = 0; j < i; j++) {
1432                                 msix = &device->msix_entries[j];
1433                                 ioat_chan = ioat_chan_by_index(device, j);
1434                                 devm_free_irq(dev, msix->vector, ioat_chan);
1435                         }
1436                         goto msix_single_vector;
1437                 }
1438         }
1439         intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
1440         goto done;
1441
1442 msix_single_vector:
1443         msix = &device->msix_entries[0];
1444         msix->entry = 0;
1445         err = pci_enable_msix(pdev, device->msix_entries, 1);
1446         if (err)
1447                 goto msi;
1448
1449         err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
1450                                "ioat-msix", device);
1451         if (err) {
1452                 pci_disable_msix(pdev);
1453                 goto msi;
1454         }
1455         goto done;
1456
1457 msi:
1458         err = pci_enable_msi(pdev);
1459         if (err)
1460                 goto intx;
1461
1462         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
1463                                "ioat-msi", device);
1464         if (err) {
1465                 pci_disable_msi(pdev);
1466                 goto intx;
1467         }
1468         goto done;
1469
1470 intx:
1471         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
1472                                IRQF_SHARED, "ioat-intx", device);
1473         if (err)
1474                 goto err_no_irq;
1475
1476 done:
1477         if (device->intr_quirk)
1478                 device->intr_quirk(device);
1479         intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
1480         writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
1481         return 0;
1482
1483 err_no_irq:
1484         /* Disable all interrupt generation */
1485         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1486         dev_err(dev, "no usable interrupts\n");
1487         return err;
1488 }
1489
1490 static void ioat_disable_interrupts(struct ioatdma_device *device)
1491 {
1492         /* Disable all interrupt generation */
1493         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1494 }
1495
1496 static int ioat_probe(struct ioatdma_device *device)
1497 {
1498         int err = -ENODEV;
1499         struct dma_device *dma = &device->common;
1500         struct pci_dev *pdev = device->pdev;
1501         struct device *dev = &pdev->dev;
1502
1503         /* DMA coherent memory pool for DMA descriptor allocations */
1504         device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1505                                            sizeof(struct ioat_dma_descriptor),
1506                                            64, 0);
1507         if (!device->dma_pool) {
1508                 err = -ENOMEM;
1509                 goto err_dma_pool;
1510         }
1511
1512         device->completion_pool = pci_pool_create("completion_pool", pdev,
1513                                                   sizeof(u64), SMP_CACHE_BYTES,
1514                                                   SMP_CACHE_BYTES);
1515         if (!device->completion_pool) {
1516                 err = -ENOMEM;
1517                 goto err_completion_pool;
1518         }
1519
1520         ioat_dma_enumerate_channels(device);
1521
1522         dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1523         dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
1524         dma->device_free_chan_resources = ioat_dma_free_chan_resources;
1525         dma->device_is_tx_complete = ioat_dma_is_complete;
1526         dma->dev = &pdev->dev;
1527
1528         dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
1529                 " %d channels, device version 0x%02x, driver version %s\n",
1530                 dma->chancnt, device->version, IOAT_DMA_VERSION);
1531
1532         if (!dma->chancnt) {
1533                 dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: "
1534                         "zero channels detected\n");
1535                 goto err_setup_interrupts;
1536         }
1537
1538         err = ioat_dma_setup_interrupts(device);
1539         if (err)
1540                 goto err_setup_interrupts;
1541
1542         err = ioat_dma_self_test(device);
1543         if (err)
1544                 goto err_self_test;
1545
1546         return 0;
1547
1548 err_self_test:
1549         ioat_disable_interrupts(device);
1550 err_setup_interrupts:
1551         pci_pool_destroy(device->completion_pool);
1552 err_completion_pool:
1553         pci_pool_destroy(device->dma_pool);
1554 err_dma_pool:
1555         return err;
1556 }
1557
1558 static int ioat_register(struct ioatdma_device *device)
1559 {
1560         int err = dma_async_device_register(&device->common);
1561
1562         if (err) {
1563                 ioat_disable_interrupts(device);
1564                 pci_pool_destroy(device->completion_pool);
1565                 pci_pool_destroy(device->dma_pool);
1566         }
1567
1568         return err;
1569 }
1570
1571 /* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
1572 static void ioat1_intr_quirk(struct ioatdma_device *device)
1573 {
1574         struct pci_dev *pdev = device->pdev;
1575         u32 dmactrl;
1576
1577         pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1578         if (pdev->msi_enabled)
1579                 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1580         else
1581                 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1582         pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1583 }
1584
1585 int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1586 {
1587         struct pci_dev *pdev = device->pdev;
1588         struct dma_device *dma;
1589         int err;
1590
1591         device->intr_quirk = ioat1_intr_quirk;
1592         dma = &device->common;
1593         dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1594         dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1595
1596         err = ioat_probe(device);
1597         if (err)
1598                 return err;
1599         ioat_set_tcp_copy_break(4096);
1600         err = ioat_register(device);
1601         if (err)
1602                 return err;
1603         if (dca)
1604                 device->dca = ioat_dca_init(pdev, device->reg_base);
1605
1606         INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1607         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1608
1609         return err;
1610 }
1611
1612 int ioat2_dma_probe(struct ioatdma_device *device, int dca)
1613 {
1614         struct pci_dev *pdev = device->pdev;
1615         struct dma_device *dma;
1616         struct dma_chan *chan;
1617         struct ioat_dma_chan *ioat_chan;
1618         int err;
1619
1620         dma = &device->common;
1621         dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1622         dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1623
1624         err = ioat_probe(device);
1625         if (err)
1626                 return err;
1627         ioat_set_tcp_copy_break(2048);
1628
1629         list_for_each_entry(chan, &dma->channels, device_node) {
1630                 ioat_chan = to_ioat_chan(chan);
1631                 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
1632                        ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
1633         }
1634
1635         err = ioat_register(device);
1636         if (err)
1637                 return err;
1638         if (dca)
1639                 device->dca = ioat2_dca_init(pdev, device->reg_base);
1640
1641         INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1642         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1643
1644         return err;
1645 }
1646
1647 int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1648 {
1649         struct pci_dev *pdev = device->pdev;
1650         struct dma_device *dma;
1651         struct dma_chan *chan;
1652         struct ioat_dma_chan *ioat_chan;
1653         int err;
1654         u16 dev_id;
1655
1656         dma = &device->common;
1657         dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1658         dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1659
1660         /* -= IOAT ver.3 workarounds =- */
1661         /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1662          * that can cause stability issues for IOAT ver.3
1663          */
1664         pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1665
1666         /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1667          * (workaround for spurious config parity error after restart)
1668          */
1669         pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1670         if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1671                 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1672
1673         err = ioat_probe(device);
1674         if (err)
1675                 return err;
1676         ioat_set_tcp_copy_break(262144);
1677
1678         list_for_each_entry(chan, &dma->channels, device_node) {
1679                 ioat_chan = to_ioat_chan(chan);
1680                 writel(IOAT_DMA_DCA_ANY_CPU,
1681                        ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
1682         }
1683
1684         err = ioat_register(device);
1685         if (err)
1686                 return err;
1687         if (dca)
1688                 device->dca = ioat3_dca_init(pdev, device->reg_base);
1689
1690         return err;
1691 }
1692
1693 void ioat_dma_remove(struct ioatdma_device *device)
1694 {
1695         struct dma_chan *chan, *_chan;
1696         struct ioat_dma_chan *ioat_chan;
1697         struct dma_device *dma = &device->common;
1698
1699         if (device->version != IOAT_VER_3_0)
1700                 cancel_delayed_work(&device->work);
1701
1702         ioat_disable_interrupts(device);
1703
1704         dma_async_device_unregister(dma);
1705
1706         pci_pool_destroy(device->dma_pool);
1707         pci_pool_destroy(device->completion_pool);
1708
1709         list_for_each_entry_safe(chan, _chan, &dma->channels, device_node) {
1710                 ioat_chan = to_ioat_chan(chan);
1711                 list_del(&chan->device_node);
1712         }
1713 }
1714