2 * sata_nv.c - NVIDIA nForce SATA
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/gfp.h>
42 #include <linux/pci.h>
43 #include <linux/init.h>
44 #include <linux/blkdev.h>
45 #include <linux/delay.h>
46 #include <linux/interrupt.h>
47 #include <linux/device.h>
48 #include <scsi/scsi_host.h>
49 #include <scsi/scsi_device.h>
50 #include <linux/libata.h>
52 #define DRV_NAME "sata_nv"
53 #define DRV_VERSION "3.5"
55 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
61 NV_PIO_MASK = ATA_PIO4,
62 NV_MWDMA_MASK = ATA_MWDMA2,
63 NV_UDMA_MASK = ATA_UDMA6,
64 NV_PORT0_SCR_REG_OFFSET = 0x00,
65 NV_PORT1_SCR_REG_OFFSET = 0x40,
67 /* INT_STATUS/ENABLE */
70 NV_INT_STATUS_CK804 = 0x440,
71 NV_INT_ENABLE_CK804 = 0x441,
73 /* INT_STATUS/ENABLE bits */
77 NV_INT_REMOVED = 0x08,
79 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
82 NV_INT_MASK = NV_INT_DEV |
83 NV_INT_ADDED | NV_INT_REMOVED,
87 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
89 // For PCI config register 20
90 NV_MCP_SATA_CFG_20 = 0x50,
91 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
92 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
93 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
94 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
95 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
97 NV_ADMA_MAX_CPBS = 32,
100 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
102 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
103 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
104 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
105 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
107 /* BAR5 offset to ADMA general registers */
109 NV_ADMA_GEN_CTL = 0x00,
110 NV_ADMA_NOTIFIER_CLEAR = 0x30,
112 /* BAR5 offset to ADMA ports */
113 NV_ADMA_PORT = 0x480,
115 /* size of ADMA port register space */
116 NV_ADMA_PORT_SIZE = 0x100,
118 /* ADMA port registers */
120 NV_ADMA_CPB_COUNT = 0x42,
121 NV_ADMA_NEXT_CPB_IDX = 0x43,
123 NV_ADMA_CPB_BASE_LOW = 0x48,
124 NV_ADMA_CPB_BASE_HIGH = 0x4C,
125 NV_ADMA_APPEND = 0x50,
126 NV_ADMA_NOTIFIER = 0x68,
127 NV_ADMA_NOTIFIER_ERROR = 0x6C,
129 /* NV_ADMA_CTL register bits */
130 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
131 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
132 NV_ADMA_CTL_GO = (1 << 7),
133 NV_ADMA_CTL_AIEN = (1 << 8),
134 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
135 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
137 /* CPB response flag bits */
138 NV_CPB_RESP_DONE = (1 << 0),
139 NV_CPB_RESP_ATA_ERR = (1 << 3),
140 NV_CPB_RESP_CMD_ERR = (1 << 4),
141 NV_CPB_RESP_CPB_ERR = (1 << 7),
143 /* CPB control flag bits */
144 NV_CPB_CTL_CPB_VALID = (1 << 0),
145 NV_CPB_CTL_QUEUE = (1 << 1),
146 NV_CPB_CTL_APRD_VALID = (1 << 2),
147 NV_CPB_CTL_IEN = (1 << 3),
148 NV_CPB_CTL_FPDMA = (1 << 4),
151 NV_APRD_WRITE = (1 << 1),
152 NV_APRD_END = (1 << 2),
153 NV_APRD_CONT = (1 << 3),
155 /* NV_ADMA_STAT flags */
156 NV_ADMA_STAT_TIMEOUT = (1 << 0),
157 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
158 NV_ADMA_STAT_HOTPLUG = (1 << 2),
159 NV_ADMA_STAT_CPBERR = (1 << 4),
160 NV_ADMA_STAT_SERROR = (1 << 5),
161 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
162 NV_ADMA_STAT_IDLE = (1 << 8),
163 NV_ADMA_STAT_LEGACY = (1 << 9),
164 NV_ADMA_STAT_STOPPED = (1 << 10),
165 NV_ADMA_STAT_DONE = (1 << 12),
166 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
167 NV_ADMA_STAT_TIMEOUT,
170 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
171 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
173 /* MCP55 reg offset */
174 NV_CTL_MCP55 = 0x400,
175 NV_INT_STATUS_MCP55 = 0x440,
176 NV_INT_ENABLE_MCP55 = 0x444,
177 NV_NCQ_REG_MCP55 = 0x448,
180 NV_INT_ALL_MCP55 = 0xffff,
181 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
182 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
184 /* SWNCQ ENABLE BITS*/
185 NV_CTL_PRI_SWNCQ = 0x02,
186 NV_CTL_SEC_SWNCQ = 0x04,
188 /* SW NCQ status bits*/
189 NV_SWNCQ_IRQ_DEV = (1 << 0),
190 NV_SWNCQ_IRQ_PM = (1 << 1),
191 NV_SWNCQ_IRQ_ADDED = (1 << 2),
192 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
194 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
195 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
196 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
197 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
199 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
200 NV_SWNCQ_IRQ_REMOVED,
204 /* ADMA Physical Region Descriptor - one SG segment */
213 enum nv_adma_regbits {
214 CMDEND = (1 << 15), /* end of command list */
215 WNB = (1 << 14), /* wait-not-BSY */
216 IGN = (1 << 13), /* ignore this entry */
217 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
218 DA2 = (1 << (2 + 8)),
219 DA1 = (1 << (1 + 8)),
220 DA0 = (1 << (0 + 8)),
223 /* ADMA Command Parameter Block
224 The first 5 SG segments are stored inside the Command Parameter Block itself.
225 If there are more than 5 segments the remainder are stored in a separate
226 memory area indicated by next_aprd. */
228 u8 resp_flags; /* 0 */
229 u8 reserved1; /* 1 */
230 u8 ctl_flags; /* 2 */
231 /* len is length of taskfile in 64 bit words */
234 u8 next_cpb_idx; /* 5 */
235 __le16 reserved2; /* 6-7 */
236 __le16 tf[12]; /* 8-31 */
237 struct nv_adma_prd aprd[5]; /* 32-111 */
238 __le64 next_aprd; /* 112-119 */
239 __le64 reserved3; /* 120-127 */
243 struct nv_adma_port_priv {
244 struct nv_adma_cpb *cpb;
246 struct nv_adma_prd *aprd;
248 void __iomem *ctl_block;
249 void __iomem *gen_block;
250 void __iomem *notifier_clear_block;
256 struct nv_host_priv {
264 unsigned int tag[ATA_MAX_QUEUE];
267 enum ncq_saw_flag_list {
268 ncq_saw_d2h = (1U << 0),
269 ncq_saw_dmas = (1U << 1),
270 ncq_saw_sdb = (1U << 2),
271 ncq_saw_backout = (1U << 3),
274 struct nv_swncq_port_priv {
275 struct ata_prd *prd; /* our SG list */
276 dma_addr_t prd_dma; /* and its DMA mapping */
277 void __iomem *sactive_block;
278 void __iomem *irq_block;
279 void __iomem *tag_block;
282 unsigned int last_issue_tag;
284 /* fifo circular queue to store deferral command */
285 struct defer_queue defer_queue;
287 /* for NCQ interrupt analysis */
292 unsigned int ncq_flags;
296 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
298 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
300 static int nv_pci_device_resume(struct pci_dev *pdev);
302 static void nv_ck804_host_stop(struct ata_host *host);
303 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
304 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
305 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
306 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
307 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
309 static int nv_hardreset(struct ata_link *link, unsigned int *class,
310 unsigned long deadline);
311 static void nv_nf2_freeze(struct ata_port *ap);
312 static void nv_nf2_thaw(struct ata_port *ap);
313 static void nv_ck804_freeze(struct ata_port *ap);
314 static void nv_ck804_thaw(struct ata_port *ap);
315 static int nv_adma_slave_config(struct scsi_device *sdev);
316 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
317 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
318 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
319 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
320 static void nv_adma_irq_clear(struct ata_port *ap);
321 static int nv_adma_port_start(struct ata_port *ap);
322 static void nv_adma_port_stop(struct ata_port *ap);
324 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
325 static int nv_adma_port_resume(struct ata_port *ap);
327 static void nv_adma_freeze(struct ata_port *ap);
328 static void nv_adma_thaw(struct ata_port *ap);
329 static void nv_adma_error_handler(struct ata_port *ap);
330 static void nv_adma_host_stop(struct ata_host *host);
331 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
332 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
334 static void nv_mcp55_thaw(struct ata_port *ap);
335 static void nv_mcp55_freeze(struct ata_port *ap);
336 static void nv_swncq_error_handler(struct ata_port *ap);
337 static int nv_swncq_slave_config(struct scsi_device *sdev);
338 static int nv_swncq_port_start(struct ata_port *ap);
339 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
340 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
341 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
342 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
343 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
345 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
346 static int nv_swncq_port_resume(struct ata_port *ap);
353 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
360 static const struct pci_device_id nv_pci_tbl[] = {
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
372 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
373 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
374 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
376 { } /* terminate list */
379 static struct pci_driver nv_pci_driver = {
381 .id_table = nv_pci_tbl,
382 .probe = nv_init_one,
384 .suspend = ata_pci_device_suspend,
385 .resume = nv_pci_device_resume,
387 .remove = ata_pci_remove_one,
390 static struct scsi_host_template nv_sht = {
391 ATA_BMDMA_SHT(DRV_NAME),
394 static struct scsi_host_template nv_adma_sht = {
395 ATA_NCQ_SHT(DRV_NAME),
396 .can_queue = NV_ADMA_MAX_CPBS,
397 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
398 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
399 .slave_configure = nv_adma_slave_config,
402 static struct scsi_host_template nv_swncq_sht = {
403 ATA_NCQ_SHT(DRV_NAME),
404 .can_queue = ATA_MAX_QUEUE,
405 .sg_tablesize = LIBATA_MAX_PRD,
406 .dma_boundary = ATA_DMA_BOUNDARY,
407 .slave_configure = nv_swncq_slave_config,
411 * NV SATA controllers have various different problems with hardreset
412 * protocol depending on the specific controller and device.
416 * bko11195 reports that link doesn't come online after hardreset on
417 * generic nv's and there have been several other similar reports on
420 * bko12351#c23 reports that warmplug on MCP61 doesn't work with
425 * bko3352 reports nf2/3 controllers can't determine device signature
426 * reliably after hardreset. The following thread reports detection
427 * failure on cold boot with the standard debouncing timing.
429 * http://thread.gmane.org/gmane.linux.ide/34098
431 * bko12176 reports that hardreset fails to bring up the link during
436 * For initial probing after boot and hot plugging, hardreset mostly
437 * works fine on CK804 but curiously, reprobing on the initial port
438 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg
439 * FIS in somewhat undeterministic way.
443 * bko12351 reports that when SWNCQ is enabled, for hotplug to work,
444 * hardreset should be used and hardreset can't report proper
445 * signature, which suggests that mcp5x is closer to nf2 as long as
446 * reset quirkiness is concerned.
448 * bko12703 reports that boot probing fails for intel SSD with
449 * hardreset. Link fails to come online. Softreset works fine.
451 * The failures are varied but the following patterns seem true for
454 * - Softreset during boot always works.
456 * - Hardreset during boot sometimes fails to bring up the link on
457 * certain comibnations and device signature acquisition is
460 * - Hardreset is often necessary after hotplug.
462 * So, preferring softreset for boot probing and error handling (as
463 * hardreset might bring down the link) but using hardreset for
464 * post-boot probing should work around the above issues in most
465 * cases. Define nv_hardreset() which only kicks in for post-boot
466 * probing and use it for all variants.
468 static struct ata_port_operations nv_generic_ops = {
469 .inherits = &ata_bmdma_port_ops,
470 .lost_interrupt = ATA_OP_NULL,
471 .scr_read = nv_scr_read,
472 .scr_write = nv_scr_write,
473 .hardreset = nv_hardreset,
476 static struct ata_port_operations nv_nf2_ops = {
477 .inherits = &nv_generic_ops,
478 .freeze = nv_nf2_freeze,
482 static struct ata_port_operations nv_ck804_ops = {
483 .inherits = &nv_generic_ops,
484 .freeze = nv_ck804_freeze,
485 .thaw = nv_ck804_thaw,
486 .host_stop = nv_ck804_host_stop,
489 static struct ata_port_operations nv_adma_ops = {
490 .inherits = &nv_ck804_ops,
492 .check_atapi_dma = nv_adma_check_atapi_dma,
493 .sff_tf_read = nv_adma_tf_read,
494 .qc_defer = ata_std_qc_defer,
495 .qc_prep = nv_adma_qc_prep,
496 .qc_issue = nv_adma_qc_issue,
497 .sff_irq_clear = nv_adma_irq_clear,
499 .freeze = nv_adma_freeze,
500 .thaw = nv_adma_thaw,
501 .error_handler = nv_adma_error_handler,
502 .post_internal_cmd = nv_adma_post_internal_cmd,
504 .port_start = nv_adma_port_start,
505 .port_stop = nv_adma_port_stop,
507 .port_suspend = nv_adma_port_suspend,
508 .port_resume = nv_adma_port_resume,
510 .host_stop = nv_adma_host_stop,
513 static struct ata_port_operations nv_swncq_ops = {
514 .inherits = &nv_generic_ops,
516 .qc_defer = ata_std_qc_defer,
517 .qc_prep = nv_swncq_qc_prep,
518 .qc_issue = nv_swncq_qc_issue,
520 .freeze = nv_mcp55_freeze,
521 .thaw = nv_mcp55_thaw,
522 .error_handler = nv_swncq_error_handler,
525 .port_suspend = nv_swncq_port_suspend,
526 .port_resume = nv_swncq_port_resume,
528 .port_start = nv_swncq_port_start,
532 irq_handler_t irq_handler;
533 struct scsi_host_template *sht;
536 #define NV_PI_PRIV(_irq_handler, _sht) \
537 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
539 static const struct ata_port_info nv_port_info[] = {
542 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
543 .pio_mask = NV_PIO_MASK,
544 .mwdma_mask = NV_MWDMA_MASK,
545 .udma_mask = NV_UDMA_MASK,
546 .port_ops = &nv_generic_ops,
547 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
551 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
552 .pio_mask = NV_PIO_MASK,
553 .mwdma_mask = NV_MWDMA_MASK,
554 .udma_mask = NV_UDMA_MASK,
555 .port_ops = &nv_nf2_ops,
556 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
560 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
561 .pio_mask = NV_PIO_MASK,
562 .mwdma_mask = NV_MWDMA_MASK,
563 .udma_mask = NV_UDMA_MASK,
564 .port_ops = &nv_ck804_ops,
565 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
569 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
570 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
571 .pio_mask = NV_PIO_MASK,
572 .mwdma_mask = NV_MWDMA_MASK,
573 .udma_mask = NV_UDMA_MASK,
574 .port_ops = &nv_adma_ops,
575 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
579 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
580 .pio_mask = NV_PIO_MASK,
581 .mwdma_mask = NV_MWDMA_MASK,
582 .udma_mask = NV_UDMA_MASK,
583 .port_ops = &nv_generic_ops,
584 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
588 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
590 .pio_mask = NV_PIO_MASK,
591 .mwdma_mask = NV_MWDMA_MASK,
592 .udma_mask = NV_UDMA_MASK,
593 .port_ops = &nv_swncq_ops,
594 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
598 MODULE_AUTHOR("NVIDIA");
599 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
600 MODULE_LICENSE("GPL");
601 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
602 MODULE_VERSION(DRV_VERSION);
604 static int adma_enabled;
605 static int swncq_enabled = 1;
606 static int msi_enabled;
608 static void nv_adma_register_mode(struct ata_port *ap)
610 struct nv_adma_port_priv *pp = ap->private_data;
611 void __iomem *mmio = pp->ctl_block;
615 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
618 status = readw(mmio + NV_ADMA_STAT);
619 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
621 status = readw(mmio + NV_ADMA_STAT);
625 ata_port_printk(ap, KERN_WARNING,
626 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
629 tmp = readw(mmio + NV_ADMA_CTL);
630 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
633 status = readw(mmio + NV_ADMA_STAT);
634 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
636 status = readw(mmio + NV_ADMA_STAT);
640 ata_port_printk(ap, KERN_WARNING,
641 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
644 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
647 static void nv_adma_mode(struct ata_port *ap)
649 struct nv_adma_port_priv *pp = ap->private_data;
650 void __iomem *mmio = pp->ctl_block;
654 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
657 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
659 tmp = readw(mmio + NV_ADMA_CTL);
660 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
662 status = readw(mmio + NV_ADMA_STAT);
663 while (((status & NV_ADMA_STAT_LEGACY) ||
664 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
666 status = readw(mmio + NV_ADMA_STAT);
670 ata_port_printk(ap, KERN_WARNING,
671 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
674 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
677 static int nv_adma_slave_config(struct scsi_device *sdev)
679 struct ata_port *ap = ata_shost_to_port(sdev->host);
680 struct nv_adma_port_priv *pp = ap->private_data;
681 struct nv_adma_port_priv *port0, *port1;
682 struct scsi_device *sdev0, *sdev1;
683 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
684 unsigned long segment_boundary, flags;
685 unsigned short sg_tablesize;
688 u32 current_reg, new_reg, config_mask;
690 rc = ata_scsi_slave_config(sdev);
692 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
693 /* Not a proper libata device, ignore */
696 spin_lock_irqsave(ap->lock, flags);
698 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
700 * NVIDIA reports that ADMA mode does not support ATAPI commands.
701 * Therefore ATAPI commands are sent through the legacy interface.
702 * However, the legacy interface only supports 32-bit DMA.
703 * Restrict DMA parameters as required by the legacy interface
704 * when an ATAPI device is connected.
706 segment_boundary = ATA_DMA_BOUNDARY;
707 /* Subtract 1 since an extra entry may be needed for padding, see
709 sg_tablesize = LIBATA_MAX_PRD - 1;
711 /* Since the legacy DMA engine is in use, we need to disable ADMA
714 nv_adma_register_mode(ap);
716 segment_boundary = NV_ADMA_DMA_BOUNDARY;
717 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
721 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, ¤t_reg);
723 if (ap->port_no == 1)
724 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
725 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
727 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
728 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
731 new_reg = current_reg | config_mask;
732 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
734 new_reg = current_reg & ~config_mask;
735 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
738 if (current_reg != new_reg)
739 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
741 port0 = ap->host->ports[0]->private_data;
742 port1 = ap->host->ports[1]->private_data;
743 sdev0 = ap->host->ports[0]->link.device[0].sdev;
744 sdev1 = ap->host->ports[1]->link.device[0].sdev;
745 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
746 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
747 /** We have to set the DMA mask to 32-bit if either port is in
748 ATAPI mode, since they are on the same PCI device which is
749 used for DMA mapping. If we set the mask we also need to set
750 the bounce limit on both ports to ensure that the block
751 layer doesn't feed addresses that cause DMA mapping to
752 choke. If either SCSI device is not allocated yet, it's OK
753 since that port will discover its correct setting when it
755 Note: Setting 32-bit mask should not fail. */
757 blk_queue_bounce_limit(sdev0->request_queue,
760 blk_queue_bounce_limit(sdev1->request_queue,
763 pci_set_dma_mask(pdev, ATA_DMA_MASK);
765 /** This shouldn't fail as it was set to this value before */
766 pci_set_dma_mask(pdev, pp->adma_dma_mask);
768 blk_queue_bounce_limit(sdev0->request_queue,
771 blk_queue_bounce_limit(sdev1->request_queue,
775 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
776 blk_queue_max_segments(sdev->request_queue, sg_tablesize);
777 ata_port_printk(ap, KERN_INFO,
778 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
779 (unsigned long long)*ap->host->dev->dma_mask,
780 segment_boundary, sg_tablesize);
782 spin_unlock_irqrestore(ap->lock, flags);
787 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
789 struct nv_adma_port_priv *pp = qc->ap->private_data;
790 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
793 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
795 /* Other than when internal or pass-through commands are executed,
796 the only time this function will be called in ADMA mode will be
797 if a command fails. In the failure case we don't care about going
798 into register mode with ADMA commands pending, as the commands will
799 all shortly be aborted anyway. We assume that NCQ commands are not
800 issued via passthrough, which is the only way that switching into
801 ADMA mode could abort outstanding commands. */
802 nv_adma_register_mode(ap);
804 ata_sff_tf_read(ap, tf);
807 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
809 unsigned int idx = 0;
811 if (tf->flags & ATA_TFLAG_ISADDR) {
812 if (tf->flags & ATA_TFLAG_LBA48) {
813 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
814 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
815 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
816 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
817 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
818 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
820 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
822 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
823 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
824 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
825 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
828 if (tf->flags & ATA_TFLAG_DEVICE)
829 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
831 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
834 cpb[idx++] = cpu_to_le16(IGN);
839 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
841 struct nv_adma_port_priv *pp = ap->private_data;
842 u8 flags = pp->cpb[cpb_num].resp_flags;
844 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
846 if (unlikely((force_err ||
847 flags & (NV_CPB_RESP_ATA_ERR |
848 NV_CPB_RESP_CMD_ERR |
849 NV_CPB_RESP_CPB_ERR)))) {
850 struct ata_eh_info *ehi = &ap->link.eh_info;
853 ata_ehi_clear_desc(ehi);
854 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
855 if (flags & NV_CPB_RESP_ATA_ERR) {
856 ata_ehi_push_desc(ehi, "ATA error");
857 ehi->err_mask |= AC_ERR_DEV;
858 } else if (flags & NV_CPB_RESP_CMD_ERR) {
859 ata_ehi_push_desc(ehi, "CMD error");
860 ehi->err_mask |= AC_ERR_DEV;
861 } else if (flags & NV_CPB_RESP_CPB_ERR) {
862 ata_ehi_push_desc(ehi, "CPB error");
863 ehi->err_mask |= AC_ERR_SYSTEM;
866 /* notifier error, but no error in CPB flags? */
867 ata_ehi_push_desc(ehi, "unknown");
868 ehi->err_mask |= AC_ERR_OTHER;
871 /* Kill all commands. EH will determine what actually failed. */
879 if (likely(flags & NV_CPB_RESP_DONE)) {
880 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
881 VPRINTK("CPB flags done, flags=0x%x\n", flags);
883 DPRINTK("Completing qc from tag %d\n", cpb_num);
886 struct ata_eh_info *ehi = &ap->link.eh_info;
887 /* Notifier bits set without a command may indicate the drive
888 is misbehaving. Raise host state machine violation on this
890 ata_port_printk(ap, KERN_ERR,
891 "notifier for tag %d with no cmd?\n",
893 ehi->err_mask |= AC_ERR_HSM;
894 ehi->action |= ATA_EH_RESET;
902 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
904 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
906 /* freeze if hotplugged */
907 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
912 /* bail out if not our interrupt */
913 if (!(irq_stat & NV_INT_DEV))
916 /* DEV interrupt w/ no active qc? */
917 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
918 ata_sff_check_status(ap);
922 /* handle interrupt */
923 return ata_sff_host_intr(ap, qc);
926 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
928 struct ata_host *host = dev_instance;
930 u32 notifier_clears[2];
932 spin_lock(&host->lock);
934 for (i = 0; i < host->n_ports; i++) {
935 struct ata_port *ap = host->ports[i];
936 struct nv_adma_port_priv *pp = ap->private_data;
937 void __iomem *mmio = pp->ctl_block;
940 u32 notifier, notifier_error;
942 notifier_clears[i] = 0;
944 /* if ADMA is disabled, use standard ata interrupt handler */
945 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
946 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
947 >> (NV_INT_PORT_SHIFT * i);
948 handled += nv_host_intr(ap, irq_stat);
952 /* if in ATA register mode, check for standard interrupts */
953 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
954 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
955 >> (NV_INT_PORT_SHIFT * i);
956 if (ata_tag_valid(ap->link.active_tag))
957 /** NV_INT_DEV indication seems unreliable
958 at times at least in ADMA mode. Force it
959 on always when a command is active, to
960 prevent losing interrupts. */
961 irq_stat |= NV_INT_DEV;
962 handled += nv_host_intr(ap, irq_stat);
965 notifier = readl(mmio + NV_ADMA_NOTIFIER);
966 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
967 notifier_clears[i] = notifier | notifier_error;
969 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
971 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
976 status = readw(mmio + NV_ADMA_STAT);
979 * Clear status. Ensure the controller sees the
980 * clearing before we start looking at any of the CPB
981 * statuses, so that any CPB completions after this
982 * point in the handler will raise another interrupt.
984 writew(status, mmio + NV_ADMA_STAT);
985 readw(mmio + NV_ADMA_STAT); /* flush posted write */
988 handled++; /* irq handled if we got here */
990 /* freeze if hotplugged or controller error */
991 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
992 NV_ADMA_STAT_HOTUNPLUG |
993 NV_ADMA_STAT_TIMEOUT |
994 NV_ADMA_STAT_SERROR))) {
995 struct ata_eh_info *ehi = &ap->link.eh_info;
997 ata_ehi_clear_desc(ehi);
998 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
999 if (status & NV_ADMA_STAT_TIMEOUT) {
1000 ehi->err_mask |= AC_ERR_SYSTEM;
1001 ata_ehi_push_desc(ehi, "timeout");
1002 } else if (status & NV_ADMA_STAT_HOTPLUG) {
1003 ata_ehi_hotplugged(ehi);
1004 ata_ehi_push_desc(ehi, "hotplug");
1005 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
1006 ata_ehi_hotplugged(ehi);
1007 ata_ehi_push_desc(ehi, "hot unplug");
1008 } else if (status & NV_ADMA_STAT_SERROR) {
1009 /* let EH analyze SError and figure out cause */
1010 ata_ehi_push_desc(ehi, "SError");
1012 ata_ehi_push_desc(ehi, "unknown");
1013 ata_port_freeze(ap);
1017 if (status & (NV_ADMA_STAT_DONE |
1018 NV_ADMA_STAT_CPBERR |
1019 NV_ADMA_STAT_CMD_COMPLETE)) {
1020 u32 check_commands = notifier_clears[i];
1023 if (status & NV_ADMA_STAT_CPBERR) {
1024 /* check all active commands */
1025 if (ata_tag_valid(ap->link.active_tag))
1026 check_commands = 1 <<
1027 ap->link.active_tag;
1029 check_commands = ap->link.sactive;
1032 /* check CPBs for completed commands */
1033 while ((pos = ffs(check_commands)) && !error) {
1035 error = nv_adma_check_cpb(ap, pos,
1036 notifier_error & (1 << pos));
1037 check_commands &= ~(1 << pos);
1042 if (notifier_clears[0] || notifier_clears[1]) {
1043 /* Note: Both notifier clear registers must be written
1044 if either is set, even if one is zero, according to NVIDIA. */
1045 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1046 writel(notifier_clears[0], pp->notifier_clear_block);
1047 pp = host->ports[1]->private_data;
1048 writel(notifier_clears[1], pp->notifier_clear_block);
1051 spin_unlock(&host->lock);
1053 return IRQ_RETVAL(handled);
1056 static void nv_adma_freeze(struct ata_port *ap)
1058 struct nv_adma_port_priv *pp = ap->private_data;
1059 void __iomem *mmio = pp->ctl_block;
1062 nv_ck804_freeze(ap);
1064 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1067 /* clear any outstanding CK804 notifications */
1068 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1069 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1071 /* Disable interrupt */
1072 tmp = readw(mmio + NV_ADMA_CTL);
1073 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1074 mmio + NV_ADMA_CTL);
1075 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1078 static void nv_adma_thaw(struct ata_port *ap)
1080 struct nv_adma_port_priv *pp = ap->private_data;
1081 void __iomem *mmio = pp->ctl_block;
1086 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1089 /* Enable interrupt */
1090 tmp = readw(mmio + NV_ADMA_CTL);
1091 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1092 mmio + NV_ADMA_CTL);
1093 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1096 static void nv_adma_irq_clear(struct ata_port *ap)
1098 struct nv_adma_port_priv *pp = ap->private_data;
1099 void __iomem *mmio = pp->ctl_block;
1100 u32 notifier_clears[2];
1102 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1103 ata_sff_irq_clear(ap);
1107 /* clear any outstanding CK804 notifications */
1108 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1109 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1111 /* clear ADMA status */
1112 writew(0xffff, mmio + NV_ADMA_STAT);
1114 /* clear notifiers - note both ports need to be written with
1115 something even though we are only clearing on one */
1116 if (ap->port_no == 0) {
1117 notifier_clears[0] = 0xFFFFFFFF;
1118 notifier_clears[1] = 0;
1120 notifier_clears[0] = 0;
1121 notifier_clears[1] = 0xFFFFFFFF;
1123 pp = ap->host->ports[0]->private_data;
1124 writel(notifier_clears[0], pp->notifier_clear_block);
1125 pp = ap->host->ports[1]->private_data;
1126 writel(notifier_clears[1], pp->notifier_clear_block);
1129 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
1131 struct nv_adma_port_priv *pp = qc->ap->private_data;
1133 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
1134 ata_sff_post_internal_cmd(qc);
1137 static int nv_adma_port_start(struct ata_port *ap)
1139 struct device *dev = ap->host->dev;
1140 struct nv_adma_port_priv *pp;
1145 struct pci_dev *pdev = to_pci_dev(dev);
1150 /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1152 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1155 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1159 rc = ata_port_start(ap);
1163 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1167 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
1168 ap->port_no * NV_ADMA_PORT_SIZE;
1169 pp->ctl_block = mmio;
1170 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
1171 pp->notifier_clear_block = pp->gen_block +
1172 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1174 /* Now that the legacy PRD and padding buffer are allocated we can
1175 safely raise the DMA mask to allocate the CPB/APRD table.
1176 These are allowed to fail since we store the value that ends up
1177 being used to set as the bounce limit in slave_config later if
1179 pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1180 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1181 pp->adma_dma_mask = *dev->dma_mask;
1183 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1184 &mem_dma, GFP_KERNEL);
1187 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1190 * First item in chunk of DMA memory:
1191 * 128-byte command parameter block (CPB)
1192 * one for each command tag
1195 pp->cpb_dma = mem_dma;
1197 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1198 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1200 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1201 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1204 * Second item: block of ADMA_SGTBL_LEN s/g entries
1207 pp->aprd_dma = mem_dma;
1209 ap->private_data = pp;
1211 /* clear any outstanding interrupt conditions */
1212 writew(0xffff, mmio + NV_ADMA_STAT);
1214 /* initialize port variables */
1215 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1217 /* clear CPB fetch count */
1218 writew(0, mmio + NV_ADMA_CPB_COUNT);
1220 /* clear GO for register mode, enable interrupt */
1221 tmp = readw(mmio + NV_ADMA_CTL);
1222 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1223 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1225 tmp = readw(mmio + NV_ADMA_CTL);
1226 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1227 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1229 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1230 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1235 static void nv_adma_port_stop(struct ata_port *ap)
1237 struct nv_adma_port_priv *pp = ap->private_data;
1238 void __iomem *mmio = pp->ctl_block;
1241 writew(0, mmio + NV_ADMA_CTL);
1245 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1247 struct nv_adma_port_priv *pp = ap->private_data;
1248 void __iomem *mmio = pp->ctl_block;
1250 /* Go to register mode - clears GO */
1251 nv_adma_register_mode(ap);
1253 /* clear CPB fetch count */
1254 writew(0, mmio + NV_ADMA_CPB_COUNT);
1256 /* disable interrupt, shut down port */
1257 writew(0, mmio + NV_ADMA_CTL);
1262 static int nv_adma_port_resume(struct ata_port *ap)
1264 struct nv_adma_port_priv *pp = ap->private_data;
1265 void __iomem *mmio = pp->ctl_block;
1268 /* set CPB block location */
1269 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1270 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1272 /* clear any outstanding interrupt conditions */
1273 writew(0xffff, mmio + NV_ADMA_STAT);
1275 /* initialize port variables */
1276 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1278 /* clear CPB fetch count */
1279 writew(0, mmio + NV_ADMA_CPB_COUNT);
1281 /* clear GO for register mode, enable interrupt */
1282 tmp = readw(mmio + NV_ADMA_CTL);
1283 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1284 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1286 tmp = readw(mmio + NV_ADMA_CTL);
1287 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1288 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1290 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1291 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1297 static void nv_adma_setup_port(struct ata_port *ap)
1299 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1300 struct ata_ioports *ioport = &ap->ioaddr;
1304 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
1306 ioport->cmd_addr = mmio;
1307 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
1308 ioport->error_addr =
1309 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1310 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1311 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1312 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1313 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1314 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
1315 ioport->status_addr =
1316 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
1317 ioport->altstatus_addr =
1318 ioport->ctl_addr = mmio + 0x20;
1321 static int nv_adma_host_init(struct ata_host *host)
1323 struct pci_dev *pdev = to_pci_dev(host->dev);
1329 /* enable ADMA on the ports */
1330 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1331 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1332 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1333 NV_MCP_SATA_CFG_20_PORT1_EN |
1334 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1336 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1338 for (i = 0; i < host->n_ports; i++)
1339 nv_adma_setup_port(host->ports[i]);
1344 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1345 struct scatterlist *sg,
1347 struct nv_adma_prd *aprd)
1350 if (qc->tf.flags & ATA_TFLAG_WRITE)
1351 flags |= NV_APRD_WRITE;
1352 if (idx == qc->n_elem - 1)
1353 flags |= NV_APRD_END;
1355 flags |= NV_APRD_CONT;
1357 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1358 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1359 aprd->flags = flags;
1360 aprd->packet_len = 0;
1363 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1365 struct nv_adma_port_priv *pp = qc->ap->private_data;
1366 struct nv_adma_prd *aprd;
1367 struct scatterlist *sg;
1372 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1373 aprd = (si < 5) ? &cpb->aprd[si] :
1374 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1375 nv_adma_fill_aprd(qc, sg, si, aprd);
1378 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1380 cpb->next_aprd = cpu_to_le64(0);
1383 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1385 struct nv_adma_port_priv *pp = qc->ap->private_data;
1387 /* ADMA engine can only be used for non-ATAPI DMA commands,
1388 or interrupt-driven no-data commands. */
1389 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1390 (qc->tf.flags & ATA_TFLAG_POLLING))
1393 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
1394 (qc->tf.protocol == ATA_PROT_NODATA))
1400 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1402 struct nv_adma_port_priv *pp = qc->ap->private_data;
1403 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1404 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1407 if (nv_adma_use_reg_mode(qc)) {
1408 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1409 (qc->flags & ATA_QCFLAG_DMAMAP));
1410 nv_adma_register_mode(qc->ap);
1411 ata_sff_qc_prep(qc);
1415 cpb->resp_flags = NV_CPB_RESP_DONE;
1422 cpb->next_cpb_idx = 0;
1424 /* turn on NCQ flags for NCQ commands */
1425 if (qc->tf.protocol == ATA_PROT_NCQ)
1426 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1428 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1430 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1432 if (qc->flags & ATA_QCFLAG_DMAMAP) {
1433 nv_adma_fill_sg(qc, cpb);
1434 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1436 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1438 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1439 until we are finished filling in all of the contents */
1441 cpb->ctl_flags = ctl_flags;
1443 cpb->resp_flags = 0;
1446 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1448 struct nv_adma_port_priv *pp = qc->ap->private_data;
1449 void __iomem *mmio = pp->ctl_block;
1450 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1454 /* We can't handle result taskfile with NCQ commands, since
1455 retrieving the taskfile switches us out of ADMA mode and would abort
1456 existing commands. */
1457 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1458 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1459 ata_dev_printk(qc->dev, KERN_ERR,
1460 "NCQ w/ RESULT_TF not allowed\n");
1461 return AC_ERR_SYSTEM;
1464 if (nv_adma_use_reg_mode(qc)) {
1465 /* use ATA register mode */
1466 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
1467 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1468 (qc->flags & ATA_QCFLAG_DMAMAP));
1469 nv_adma_register_mode(qc->ap);
1470 return ata_sff_qc_issue(qc);
1472 nv_adma_mode(qc->ap);
1474 /* write append register, command tag in lower 8 bits
1475 and (number of cpbs to append -1) in top 8 bits */
1478 if (curr_ncq != pp->last_issue_ncq) {
1479 /* Seems to need some delay before switching between NCQ and
1480 non-NCQ commands, else we get command timeouts and such. */
1482 pp->last_issue_ncq = curr_ncq;
1485 writew(qc->tag, mmio + NV_ADMA_APPEND);
1487 DPRINTK("Issued tag %u\n", qc->tag);
1492 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1494 struct ata_host *host = dev_instance;
1496 unsigned int handled = 0;
1497 unsigned long flags;
1499 spin_lock_irqsave(&host->lock, flags);
1501 for (i = 0; i < host->n_ports; i++) {
1502 struct ata_port *ap = host->ports[i];
1503 struct ata_queued_cmd *qc;
1505 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1506 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {
1507 handled += ata_sff_host_intr(ap, qc);
1510 * No request pending? Clear interrupt status
1511 * anyway, in case there's one pending.
1513 ap->ops->sff_check_status(ap);
1517 spin_unlock_irqrestore(&host->lock, flags);
1519 return IRQ_RETVAL(handled);
1522 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1526 for (i = 0; i < host->n_ports; i++) {
1527 handled += nv_host_intr(host->ports[i], irq_stat);
1528 irq_stat >>= NV_INT_PORT_SHIFT;
1531 return IRQ_RETVAL(handled);
1534 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1536 struct ata_host *host = dev_instance;
1540 spin_lock(&host->lock);
1541 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1542 ret = nv_do_interrupt(host, irq_stat);
1543 spin_unlock(&host->lock);
1548 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1550 struct ata_host *host = dev_instance;
1554 spin_lock(&host->lock);
1555 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1556 ret = nv_do_interrupt(host, irq_stat);
1557 spin_unlock(&host->lock);
1562 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1564 if (sc_reg > SCR_CONTROL)
1567 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
1571 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1573 if (sc_reg > SCR_CONTROL)
1576 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
1580 static int nv_hardreset(struct ata_link *link, unsigned int *class,
1581 unsigned long deadline)
1583 struct ata_eh_context *ehc = &link->eh_context;
1585 /* Do hardreset iff it's post-boot probing, please read the
1586 * comment above port ops for details.
1588 if (!(link->ap->pflags & ATA_PFLAG_LOADING) &&
1589 !ata_dev_enabled(link->device))
1590 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1593 const unsigned long *timing = sata_ehc_deb_timing(ehc);
1596 if (!(ehc->i.flags & ATA_EHI_QUIET))
1597 ata_link_printk(link, KERN_INFO, "nv: skipping "
1598 "hardreset on occupied port\n");
1600 /* make sure the link is online */
1601 rc = sata_link_resume(link, timing, deadline);
1602 /* whine about phy resume failure but proceed */
1603 if (rc && rc != -EOPNOTSUPP)
1604 ata_link_printk(link, KERN_WARNING, "failed to resume "
1605 "link (errno=%d)\n", rc);
1608 /* device signature acquisition is unreliable */
1612 static void nv_nf2_freeze(struct ata_port *ap)
1614 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1615 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1618 mask = ioread8(scr_addr + NV_INT_ENABLE);
1619 mask &= ~(NV_INT_ALL << shift);
1620 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1623 static void nv_nf2_thaw(struct ata_port *ap)
1625 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1626 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1629 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1631 mask = ioread8(scr_addr + NV_INT_ENABLE);
1632 mask |= (NV_INT_MASK << shift);
1633 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1636 static void nv_ck804_freeze(struct ata_port *ap)
1638 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1639 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1642 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1643 mask &= ~(NV_INT_ALL << shift);
1644 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1647 static void nv_ck804_thaw(struct ata_port *ap)
1649 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1650 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1653 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1655 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1656 mask |= (NV_INT_MASK << shift);
1657 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1660 static void nv_mcp55_freeze(struct ata_port *ap)
1662 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1663 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1666 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1668 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1669 mask &= ~(NV_INT_ALL_MCP55 << shift);
1670 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1674 static void nv_mcp55_thaw(struct ata_port *ap)
1676 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1677 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1680 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1682 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1683 mask |= (NV_INT_MASK_MCP55 << shift);
1684 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1688 static void nv_adma_error_handler(struct ata_port *ap)
1690 struct nv_adma_port_priv *pp = ap->private_data;
1691 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1692 void __iomem *mmio = pp->ctl_block;
1696 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
1697 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1698 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1699 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1700 u32 status = readw(mmio + NV_ADMA_STAT);
1701 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1702 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1704 ata_port_printk(ap, KERN_ERR,
1705 "EH in ADMA mode, notifier 0x%X "
1706 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1707 "next cpb count 0x%X next cpb idx 0x%x\n",
1708 notifier, notifier_error, gen_ctl, status,
1709 cpb_count, next_cpb_idx);
1711 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
1712 struct nv_adma_cpb *cpb = &pp->cpb[i];
1713 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
1714 ap->link.sactive & (1 << i))
1715 ata_port_printk(ap, KERN_ERR,
1716 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1717 i, cpb->ctl_flags, cpb->resp_flags);
1721 /* Push us back into port register mode for error handling. */
1722 nv_adma_register_mode(ap);
1724 /* Mark all of the CPBs as invalid to prevent them from
1726 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
1727 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1729 /* clear CPB fetch count */
1730 writew(0, mmio + NV_ADMA_CPB_COUNT);
1733 tmp = readw(mmio + NV_ADMA_CTL);
1734 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1735 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1737 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1738 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1741 ata_sff_error_handler(ap);
1744 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1746 struct nv_swncq_port_priv *pp = ap->private_data;
1747 struct defer_queue *dq = &pp->defer_queue;
1750 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1751 dq->defer_bits |= (1 << qc->tag);
1752 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1755 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1757 struct nv_swncq_port_priv *pp = ap->private_data;
1758 struct defer_queue *dq = &pp->defer_queue;
1761 if (dq->head == dq->tail) /* null queue */
1764 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1765 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1766 WARN_ON(!(dq->defer_bits & (1 << tag)));
1767 dq->defer_bits &= ~(1 << tag);
1769 return ata_qc_from_tag(ap, tag);
1772 static void nv_swncq_fis_reinit(struct ata_port *ap)
1774 struct nv_swncq_port_priv *pp = ap->private_data;
1777 pp->dmafis_bits = 0;
1778 pp->sdbfis_bits = 0;
1782 static void nv_swncq_pp_reinit(struct ata_port *ap)
1784 struct nv_swncq_port_priv *pp = ap->private_data;
1785 struct defer_queue *dq = &pp->defer_queue;
1791 pp->last_issue_tag = ATA_TAG_POISON;
1792 nv_swncq_fis_reinit(ap);
1795 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1797 struct nv_swncq_port_priv *pp = ap->private_data;
1799 writew(fis, pp->irq_block);
1802 static void __ata_bmdma_stop(struct ata_port *ap)
1804 struct ata_queued_cmd qc;
1807 ata_bmdma_stop(&qc);
1810 static void nv_swncq_ncq_stop(struct ata_port *ap)
1812 struct nv_swncq_port_priv *pp = ap->private_data;
1817 ata_port_printk(ap, KERN_ERR,
1818 "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1819 ap->qc_active, ap->link.sactive);
1820 ata_port_printk(ap, KERN_ERR,
1821 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1822 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1823 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1824 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1826 ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1827 ap->ops->sff_check_status(ap),
1828 ioread8(ap->ioaddr.error_addr));
1830 sactive = readl(pp->sactive_block);
1831 done_mask = pp->qc_active ^ sactive;
1833 ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1834 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1836 if (pp->qc_active & (1 << i))
1838 else if (done_mask & (1 << i))
1843 ata_port_printk(ap, KERN_ERR,
1844 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1845 (pp->dhfis_bits >> i) & 0x1,
1846 (pp->dmafis_bits >> i) & 0x1,
1847 (pp->sdbfis_bits >> i) & 0x1,
1848 (sactive >> i) & 0x1,
1849 (err ? "error! tag doesn't exit" : " "));
1852 nv_swncq_pp_reinit(ap);
1853 ap->ops->sff_irq_clear(ap);
1854 __ata_bmdma_stop(ap);
1855 nv_swncq_irq_clear(ap, 0xffff);
1858 static void nv_swncq_error_handler(struct ata_port *ap)
1860 struct ata_eh_context *ehc = &ap->link.eh_context;
1862 if (ap->link.sactive) {
1863 nv_swncq_ncq_stop(ap);
1864 ehc->i.action |= ATA_EH_RESET;
1867 ata_sff_error_handler(ap);
1871 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1873 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1877 writel(~0, mmio + NV_INT_STATUS_MCP55);
1880 writel(0, mmio + NV_INT_ENABLE_MCP55);
1883 tmp = readl(mmio + NV_CTL_MCP55);
1884 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1885 writel(tmp, mmio + NV_CTL_MCP55);
1890 static int nv_swncq_port_resume(struct ata_port *ap)
1892 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1896 writel(~0, mmio + NV_INT_STATUS_MCP55);
1899 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1902 tmp = readl(mmio + NV_CTL_MCP55);
1903 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1909 static void nv_swncq_host_init(struct ata_host *host)
1912 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1913 struct pci_dev *pdev = to_pci_dev(host->dev);
1916 /* disable ECO 398 */
1917 pci_read_config_byte(pdev, 0x7f, ®val);
1918 regval &= ~(1 << 7);
1919 pci_write_config_byte(pdev, 0x7f, regval);
1922 tmp = readl(mmio + NV_CTL_MCP55);
1923 VPRINTK("HOST_CTL:0x%X\n", tmp);
1924 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1926 /* enable irq intr */
1927 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1928 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1929 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1931 /* clear port irq */
1932 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1935 static int nv_swncq_slave_config(struct scsi_device *sdev)
1937 struct ata_port *ap = ata_shost_to_port(sdev->host);
1938 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1939 struct ata_device *dev;
1942 u8 check_maxtor = 0;
1943 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1945 rc = ata_scsi_slave_config(sdev);
1946 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1947 /* Not a proper libata device, ignore */
1950 dev = &ap->link.device[sdev->id];
1951 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1954 /* if MCP51 and Maxtor, then disable ncq */
1955 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1956 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1959 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1960 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1961 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1962 pci_read_config_byte(pdev, 0x8, &rev);
1970 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1972 if (strncmp(model_num, "Maxtor", 6) == 0) {
1973 ata_scsi_change_queue_depth(sdev, 1, SCSI_QDEPTH_DEFAULT);
1974 ata_dev_printk(dev, KERN_NOTICE,
1975 "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1981 static int nv_swncq_port_start(struct ata_port *ap)
1983 struct device *dev = ap->host->dev;
1984 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1985 struct nv_swncq_port_priv *pp;
1988 rc = ata_port_start(ap);
1992 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1996 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1997 &pp->prd_dma, GFP_KERNEL);
2000 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
2002 ap->private_data = pp;
2003 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
2004 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
2005 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
2010 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
2012 if (qc->tf.protocol != ATA_PROT_NCQ) {
2013 ata_sff_qc_prep(qc);
2017 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
2020 nv_swncq_fill_sg(qc);
2023 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
2025 struct ata_port *ap = qc->ap;
2026 struct scatterlist *sg;
2027 struct nv_swncq_port_priv *pp = ap->private_data;
2028 struct ata_prd *prd;
2029 unsigned int si, idx;
2031 prd = pp->prd + ATA_MAX_PRD * qc->tag;
2034 for_each_sg(qc->sg, sg, qc->n_elem, si) {
2038 addr = (u32)sg_dma_address(sg);
2039 sg_len = sg_dma_len(sg);
2042 offset = addr & 0xffff;
2044 if ((offset + sg_len) > 0x10000)
2045 len = 0x10000 - offset;
2047 prd[idx].addr = cpu_to_le32(addr);
2048 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2056 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2059 static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2060 struct ata_queued_cmd *qc)
2062 struct nv_swncq_port_priv *pp = ap->private_data;
2069 writel((1 << qc->tag), pp->sactive_block);
2070 pp->last_issue_tag = qc->tag;
2071 pp->dhfis_bits &= ~(1 << qc->tag);
2072 pp->dmafis_bits &= ~(1 << qc->tag);
2073 pp->qc_active |= (0x1 << qc->tag);
2075 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
2076 ap->ops->sff_exec_command(ap, &qc->tf);
2078 DPRINTK("Issued tag %u\n", qc->tag);
2083 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2085 struct ata_port *ap = qc->ap;
2086 struct nv_swncq_port_priv *pp = ap->private_data;
2088 if (qc->tf.protocol != ATA_PROT_NCQ)
2089 return ata_sff_qc_issue(qc);
2094 nv_swncq_issue_atacmd(ap, qc);
2096 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2101 static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2104 struct ata_eh_info *ehi = &ap->link.eh_info;
2106 ata_ehi_clear_desc(ehi);
2108 /* AHCI needs SError cleared; otherwise, it might lock up */
2109 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2110 sata_scr_write(&ap->link, SCR_ERROR, serror);
2112 /* analyze @irq_stat */
2113 if (fis & NV_SWNCQ_IRQ_ADDED)
2114 ata_ehi_push_desc(ehi, "hot plug");
2115 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2116 ata_ehi_push_desc(ehi, "hot unplug");
2118 ata_ehi_hotplugged(ehi);
2120 /* okay, let's hand over to EH */
2121 ehi->serror |= serror;
2123 ata_port_freeze(ap);
2126 static int nv_swncq_sdbfis(struct ata_port *ap)
2128 struct ata_queued_cmd *qc;
2129 struct nv_swncq_port_priv *pp = ap->private_data;
2130 struct ata_eh_info *ehi = &ap->link.eh_info;
2138 host_stat = ap->ops->bmdma_status(ap);
2139 if (unlikely(host_stat & ATA_DMA_ERR)) {
2140 /* error when transfering data to/from memory */
2141 ata_ehi_clear_desc(ehi);
2142 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2143 ehi->err_mask |= AC_ERR_HOST_BUS;
2144 ehi->action |= ATA_EH_RESET;
2148 ap->ops->sff_irq_clear(ap);
2149 __ata_bmdma_stop(ap);
2151 sactive = readl(pp->sactive_block);
2152 done_mask = pp->qc_active ^ sactive;
2154 if (unlikely(done_mask & sactive)) {
2155 ata_ehi_clear_desc(ehi);
2156 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2157 "(%08x->%08x)", pp->qc_active, sactive);
2158 ehi->err_mask |= AC_ERR_HSM;
2159 ehi->action |= ATA_EH_RESET;
2162 for (i = 0; i < ATA_MAX_QUEUE; i++) {
2163 if (!(done_mask & (1 << i)))
2166 qc = ata_qc_from_tag(ap, i);
2168 ata_qc_complete(qc);
2169 pp->qc_active &= ~(1 << i);
2170 pp->dhfis_bits &= ~(1 << i);
2171 pp->dmafis_bits &= ~(1 << i);
2172 pp->sdbfis_bits |= (1 << i);
2177 if (!ap->qc_active) {
2179 nv_swncq_pp_reinit(ap);
2183 if (pp->qc_active & pp->dhfis_bits)
2186 if ((pp->ncq_flags & ncq_saw_backout) ||
2187 (pp->qc_active ^ pp->dhfis_bits))
2188 /* if the controller cann't get a device to host register FIS,
2189 * The driver needs to reissue the new command.
2193 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2194 "SWNCQ:qc_active 0x%X defer_bits %X "
2195 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2196 ap->print_id, ap->qc_active, pp->qc_active,
2197 pp->defer_queue.defer_bits, pp->dhfis_bits,
2198 pp->dmafis_bits, pp->last_issue_tag);
2200 nv_swncq_fis_reinit(ap);
2203 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2204 nv_swncq_issue_atacmd(ap, qc);
2208 if (pp->defer_queue.defer_bits) {
2209 /* send deferral queue command */
2210 qc = nv_swncq_qc_from_dq(ap);
2211 WARN_ON(qc == NULL);
2212 nv_swncq_issue_atacmd(ap, qc);
2218 static inline u32 nv_swncq_tag(struct ata_port *ap)
2220 struct nv_swncq_port_priv *pp = ap->private_data;
2223 tag = readb(pp->tag_block) >> 2;
2224 return (tag & 0x1f);
2227 static int nv_swncq_dmafis(struct ata_port *ap)
2229 struct ata_queued_cmd *qc;
2233 struct nv_swncq_port_priv *pp = ap->private_data;
2235 __ata_bmdma_stop(ap);
2236 tag = nv_swncq_tag(ap);
2238 DPRINTK("dma setup tag 0x%x\n", tag);
2239 qc = ata_qc_from_tag(ap, tag);
2244 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2246 /* load PRD table addr. */
2247 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2248 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2250 /* specify data direction, triple-check start bit is clear */
2251 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2252 dmactl &= ~ATA_DMA_WR;
2254 dmactl |= ATA_DMA_WR;
2256 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2261 static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2263 struct nv_swncq_port_priv *pp = ap->private_data;
2264 struct ata_queued_cmd *qc;
2265 struct ata_eh_info *ehi = &ap->link.eh_info;
2270 ata_stat = ap->ops->sff_check_status(ap);
2271 nv_swncq_irq_clear(ap, fis);
2275 if (ap->pflags & ATA_PFLAG_FROZEN)
2278 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2279 nv_swncq_hotplug(ap, fis);
2286 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
2288 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
2290 if (ata_stat & ATA_ERR) {
2291 ata_ehi_clear_desc(ehi);
2292 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2293 ehi->err_mask |= AC_ERR_DEV;
2294 ehi->serror |= serror;
2295 ehi->action |= ATA_EH_RESET;
2296 ata_port_freeze(ap);
2300 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2301 /* If the IRQ is backout, driver must issue
2302 * the new command again some time later.
2304 pp->ncq_flags |= ncq_saw_backout;
2307 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2308 pp->ncq_flags |= ncq_saw_sdb;
2309 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2310 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2311 ap->print_id, pp->qc_active, pp->dhfis_bits,
2312 pp->dmafis_bits, readl(pp->sactive_block));
2313 rc = nv_swncq_sdbfis(ap);
2318 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2319 /* The interrupt indicates the new command
2320 * was transmitted correctly to the drive.
2322 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2323 pp->ncq_flags |= ncq_saw_d2h;
2324 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2325 ata_ehi_push_desc(ehi, "illegal fis transaction");
2326 ehi->err_mask |= AC_ERR_HSM;
2327 ehi->action |= ATA_EH_RESET;
2331 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2332 !(pp->ncq_flags & ncq_saw_dmas)) {
2333 ata_stat = ap->ops->sff_check_status(ap);
2334 if (ata_stat & ATA_BUSY)
2337 if (pp->defer_queue.defer_bits) {
2338 DPRINTK("send next command\n");
2339 qc = nv_swncq_qc_from_dq(ap);
2340 nv_swncq_issue_atacmd(ap, qc);
2345 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2346 /* program the dma controller with appropriate PRD buffers
2347 * and start the DMA transfer for requested command.
2349 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2350 pp->ncq_flags |= ncq_saw_dmas;
2351 rc = nv_swncq_dmafis(ap);
2357 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2358 ata_port_freeze(ap);
2362 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2364 struct ata_host *host = dev_instance;
2366 unsigned int handled = 0;
2367 unsigned long flags;
2370 spin_lock_irqsave(&host->lock, flags);
2372 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2374 for (i = 0; i < host->n_ports; i++) {
2375 struct ata_port *ap = host->ports[i];
2377 if (ap->link.sactive) {
2378 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2381 if (irq_stat) /* reserve Hotplug */
2382 nv_swncq_irq_clear(ap, 0xfff0);
2384 handled += nv_host_intr(ap, (u8)irq_stat);
2386 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2389 spin_unlock_irqrestore(&host->lock, flags);
2391 return IRQ_RETVAL(handled);
2394 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
2396 static int printed_version;
2397 const struct ata_port_info *ppi[] = { NULL, NULL };
2398 struct nv_pi_priv *ipriv;
2399 struct ata_host *host;
2400 struct nv_host_priv *hpriv;
2404 unsigned long type = ent->driver_data;
2406 // Make sure this is a SATA controller by counting the number of bars
2407 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2408 // it's an IDE controller and we ignore it.
2409 for (bar = 0; bar < 6; bar++)
2410 if (pci_resource_start(pdev, bar) == 0)
2413 if (!printed_version++)
2414 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
2416 rc = pcim_enable_device(pdev);
2420 /* determine type and allocate host */
2421 if (type == CK804 && adma_enabled) {
2422 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2424 } else if (type == MCP5x && swncq_enabled) {
2425 dev_printk(KERN_NOTICE, &pdev->dev, "Using SWNCQ mode\n");
2429 ppi[0] = &nv_port_info[type];
2430 ipriv = ppi[0]->private_data;
2431 rc = ata_pci_sff_prepare_host(pdev, ppi, &host);
2435 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2439 host->private_data = hpriv;
2441 /* request and iomap NV_MMIO_BAR */
2442 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2446 /* configure SCR access */
2447 base = host->iomap[NV_MMIO_BAR];
2448 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2449 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
2451 /* enable SATA space for CK804 */
2452 if (type >= CK804) {
2455 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
2456 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2457 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2462 rc = nv_adma_host_init(host);
2465 } else if (type == SWNCQ)
2466 nv_swncq_host_init(host);
2469 dev_printk(KERN_NOTICE, &pdev->dev, "Using MSI\n");
2470 pci_enable_msi(pdev);
2473 pci_set_master(pdev);
2474 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht);
2478 static int nv_pci_device_resume(struct pci_dev *pdev)
2480 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2481 struct nv_host_priv *hpriv = host->private_data;
2484 rc = ata_pci_device_do_resume(pdev);
2488 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2489 if (hpriv->type >= CK804) {
2492 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
2493 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2494 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2496 if (hpriv->type == ADMA) {
2498 struct nv_adma_port_priv *pp;
2499 /* enable/disable ADMA on the ports appropriately */
2500 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2502 pp = host->ports[0]->private_data;
2503 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2504 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2505 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2507 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
2508 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2509 pp = host->ports[1]->private_data;
2510 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2511 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2512 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2514 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
2515 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2517 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2521 ata_host_resume(host);
2527 static void nv_ck804_host_stop(struct ata_host *host)
2529 struct pci_dev *pdev = to_pci_dev(host->dev);
2532 /* disable SATA space for CK804 */
2533 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
2534 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2535 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2538 static void nv_adma_host_stop(struct ata_host *host)
2540 struct pci_dev *pdev = to_pci_dev(host->dev);
2543 /* disable ADMA on the ports */
2544 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2545 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2546 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2547 NV_MCP_SATA_CFG_20_PORT1_EN |
2548 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2550 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2552 nv_ck804_host_stop(host);
2555 static int __init nv_init(void)
2557 return pci_register_driver(&nv_pci_driver);
2560 static void __exit nv_exit(void)
2562 pci_unregister_driver(&nv_pci_driver);
2565 module_init(nv_init);
2566 module_exit(nv_exit);
2567 module_param_named(adma, adma_enabled, bool, 0444);
2568 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
2569 module_param_named(swncq, swncq_enabled, bool, 0444);
2570 MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
2571 module_param_named(msi, msi_enabled, bool, 0444);
2572 MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)");