[safe/jmp/linux-2.6] / drivers / scsi / mvsas.c

/*
        mvsas.c - Marvell 88SE6440 SAS/SATA support

        Copyright 2007 Red Hat, Inc.

        This program is free software; you can redistribute it and/or
        modify it under the terms of the GNU General Public License as
        published by the Free Software Foundation; either version 2,
        or (at your option) any later version.

        This program is distributed in the hope that it will be useful,
        but WITHOUT ANY WARRANTY; without even the implied warranty
        of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
        See the GNU General Public License for more details.

        You should have received a copy of the GNU General Public
        License along with this program; see the file COPYING.  If not,
        write to the Free Software Foundation, 675 Mass Ave, Cambridge,
        MA 02139, USA.

        ---------------------------------------------------------------

        Random notes:
        * hardware supports controlling the endian-ness of data
          structures.  this permits elimination of all the le32_to_cpu()
          and cpu_to_le32() conversions.

 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <scsi/libsas.h>
#include <asm/io.h>

#define DRV_NAME "mvsas"
#define DRV_VERSION "0.1"

#define mr32(reg)       readl(regs + MVS_##reg)
#define mw32(reg,val)   writel((val), regs + MVS_##reg)
#define mw32_f(reg,val) do {            \
        writel((val), regs + MVS_##reg);        \
        readl(regs + MVS_##reg);                \
        } while (0)

/* driver compile-time configuration */
enum driver_configuration {
        MVS_TX_RING_SZ          = 1024, /* TX ring size (12-bit) */
        MVS_RX_RING_SZ          = 1024, /* RX ring size (12-bit) */
                                        /* software requires power-of-2
                                           ring size */

        MVS_SLOTS               = 512,  /* command slots */
        MVS_SLOT_BUF_SZ         = 8192, /* cmd tbl + IU + status + PRD */
        MVS_SSP_CMD_SZ          = 64,   /* SSP command table buffer size */
        MVS_ATA_CMD_SZ          = 128,  /* SATA command table buffer size */
        MVS_OAF_SZ              = 64,   /* Open address frame buffer size */

        MVS_RX_FIS_COUNT        = 17,   /* Optional rx'd FISs (max 17) */
};

/* unchangeable hardware details */
enum hardware_details {
        MVS_MAX_PHYS            = 8,    /* max. possible phys */
        MVS_MAX_PORTS           = 8,    /* max. possible ports */
        MVS_RX_FISL_SZ          = 0x400 + (MVS_RX_FIS_COUNT * 0x100),
};

/* peripheral registers (BAR2) */
enum peripheral_registers {
        SPI_CTL                 = 0x10, /* EEPROM control */
        SPI_CMD                 = 0x14, /* EEPROM command */
        SPI_DATA                = 0x18, /* EEPROM data */
};

enum peripheral_register_bits {
        TWSI_RDY                = (1U << 7),    /* EEPROM interface ready */
        TWSI_RD                 = (1U << 4),    /* EEPROM read access */

        SPI_ADDR_MASK           = 0x3ffff,      /* bits 17:0 */
};

/* enhanced mode registers (BAR4) */
enum hw_registers {
        MVS_GBL_CTL             = 0x04,  /* global control */
        MVS_GBL_INT_STAT        = 0x08,  /* global irq status */
        MVS_GBL_PI              = 0x0C,  /* ports implemented bitmask */
        MVS_GBL_PORT_TYPE       = 0x00,  /* port type */

        MVS_CTL                 = 0x100, /* SAS/SATA port configuration */
        MVS_PCS                 = 0x104, /* SAS/SATA port control/status */
        MVS_CMD_LIST_LO         = 0x108, /* cmd list addr */
        MVS_CMD_LIST_HI         = 0x10C,
        MVS_RX_FIS_LO           = 0x110, /* RX FIS list addr */
        MVS_RX_FIS_HI           = 0x114,

        MVS_TX_CFG              = 0x120, /* TX configuration */
        MVS_TX_LO               = 0x124, /* TX (delivery) ring addr */
        MVS_TX_HI               = 0x128,

        MVS_RX_PROD_IDX         = 0x12C, /* RX producer pointer */
        MVS_RX_CONS_IDX         = 0x130, /* RX consumer pointer (RO) */
        MVS_RX_CFG              = 0x134, /* RX configuration */
        MVS_RX_LO               = 0x138, /* RX (completion) ring addr */
        MVS_RX_HI               = 0x13C,

        MVS_INT_COAL            = 0x148, /* Int coalescing config */
        MVS_INT_COAL_TMOUT      = 0x14C, /* Int coalescing timeout */
        MVS_INT_STAT            = 0x150, /* Central int status */
        MVS_INT_MASK            = 0x154, /* Central int enable */
        MVS_INT_STAT_SRS        = 0x158, /* SATA register set status */

                                         /* ports 1-3 follow after this */
        MVS_P0_INT_STAT         = 0x160, /* port0 interrupt status */
        MVS_P0_INT_MASK         = 0x164, /* port0 interrupt mask */

                                         /* ports 1-3 follow after this */
        MVS_P0_SER_CTLSTAT      = 0x180, /* port0 serial control/status */

        MVS_CMD_ADDR            = 0x1B8, /* Command register port (addr) */
        MVS_CMD_DATA            = 0x1BC, /* Command register port (data) */

                                         /* ports 1-3 follow after this */
        MVS_P0_CFG_ADDR         = 0x1C0, /* port0 phy register address */
        MVS_P0_CFG_DATA         = 0x1C4, /* port0 phy register data */
};

enum hw_register_bits {
        /* MVS_GBL_CTL */
        INT_EN                  = (1U << 1),    /* Global int enable */
        HBA_RST                 = (1U << 0),    /* HBA reset */

        /* MVS_GBL_INT_STAT */
        INT_XOR                 = (1U << 4),    /* XOR engine event */
        INT_SAS_SATA            = (1U << 0),    /* SAS/SATA event */

        /* MVS_GBL_PORT_TYPE */                 /* shl for ports 1-3 */
        SATA_TARGET             = (1U << 16),   /* port0 SATA target enable */
        AUTO_DET                = (1U << 8),    /* port0 SAS/SATA autodetect */
        SAS_MODE                = (1U << 0),    /* port0 SAS(1), SATA(0) mode */
                                                /* SAS_MODE value may be
                                                 * dictated (in hw) by values
                                                 * of SATA_TARGET & AUTO_DET
                                                 */

        /* MVS_TX_CFG */
        TX_EN                   = (1U << 16),   /* Enable TX */
        TX_RING_SZ_MASK         = 0xfff,        /* TX ring size, bits 11:0 */

        /* MVS_RX_CFG */
        RX_EN                   = (1U << 16),   /* Enable RX */
        RX_RING_SZ_MASK         = 0xfff,        /* RX ring size, bits 11:0 */

        /* MVS_INT_COAL */
        COAL_EN                 = (1U << 16),   /* Enable int coalescing */

        /* MVS_INT_STAT, MVS_INT_MASK */
        CINT_I2C                = (1U << 31),   /* I2C event */
        CINT_SW0                = (1U << 30),   /* software event 0 */
        CINT_SW1                = (1U << 29),   /* software event 1 */
        CINT_PRD_BC             = (1U << 28),   /* PRD BC err for read cmd */
        CINT_DMA_PCIE           = (1U << 27),   /* DMA to PCIE timeout */
        CINT_MEM                = (1U << 26),   /* int mem parity err */
        CINT_I2C_SLAVE          = (1U << 25),   /* slave I2C event */
        CINT_SRS                = (1U << 3),    /* SRS event */
        CINT_CI_STOP            = (1U << 10),   /* cmd issue stopped */
        CINT_DONE               = (1U << 0),    /* cmd completion */

                                                /* shl for ports 1-3 */
        CINT_PORT_STOPPED       = (1U << 16),   /* port0 stopped */
        CINT_PORT               = (1U << 8),    /* port0 event */

        /* TX (delivery) ring bits */
        TXQ_CMD_SHIFT           = 29,
        TXQ_CMD_SSP             = 1,            /* SSP protocol */
        TXQ_CMD_SMP             = 2,            /* SMP protocol */
        TXQ_CMD_STP             = 3,            /* STP/SATA protocol */
        TXQ_CMD_SSP_FREE_LIST   = 4,            /* add to SSP targ free list */
        TXQ_CMD_SLOT_RESET      = 7,            /* reset command slot */
        TXQ_MODE_I              = (1U << 28),   /* mode: 0=target,1=initiator */
        TXQ_PRIO_HI             = (1U << 27),   /* priority: 0=normal, 1=high */
        TXQ_SRS_SHIFT           = 20,           /* SATA register set */
        TXQ_SRS_MASK            = 0x7f,
        TXQ_PHY_SHIFT           = 12,           /* PHY bitmap */
        TXQ_PHY_MASK            = 0xff,
        TXQ_SLOT_MASK           = 0xfff,        /* slot number */

        /* RX (completion) ring bits */
        RXQ_GOOD                = (1U << 23),   /* Response good */
        RXQ_SLOT_RESET          = (1U << 21),   /* Slot reset complete */
        RXQ_CMD_RX              = (1U << 20),   /* target cmd received */
        RXQ_ATTN                = (1U << 19),   /* attention */
        RXQ_RSP                 = (1U << 18),   /* response frame xfer'd */
        RXQ_ERR                 = (1U << 17),   /* err info rec xfer'd */
        RXQ_DONE                = (1U << 16),   /* cmd complete */
        RXQ_SLOT_MASK           = 0xfff,        /* slot number */

        /* mvs_cmd_hdr bits */
        MCH_PRD_LEN_SHIFT       = 16,           /* 16-bit PRD table len */
        MCH_SSP_FR_TYPE_SHIFT   = 13,           /* SSP frame type */

                                                /* SSP initiator only */
        MCH_SSP_FR_CMD          = 0x0,          /* COMMAND frame */

                                                /* SSP initiator or target */
        MCH_SSP_FR_TASK         = 0x1,          /* TASK frame */

                                                /* SSP target only */
        MCH_SSP_FR_XFER_RDY     = 0x4,          /* XFER_RDY frame */
        MCH_SSP_FR_RESP         = 0x5,          /* RESPONSE frame */
        MCH_SSP_FR_READ         = 0x6,          /* Read DATA frame(s) */
        MCH_SSP_FR_READ_RESP    = 0x7,          /* ditto, plus RESPONSE */

        MCH_PASSTHRU            = (1U << 12),   /* pass-through (SSP) */
        MCH_FBURST              = (1U << 11),   /* first burst (SSP) */
        MCH_CHK_LEN             = (1U << 10),   /* chk xfer len (SSP) */
        MCH_RETRY               = (1U << 9),    /* tport layer retry (SSP) */
        MCH_PROTECTION          = (1U << 8),    /* protection info rec (SSP) */
        MCH_RESET               = (1U << 7),    /* Reset (STP/SATA) */
        MCH_FPDMA               = (1U << 6),    /* First party DMA (STP/SATA) */
        MCH_ATAPI               = (1U << 5),    /* ATAPI (STP/SATA) */
        MCH_BIST                = (1U << 4),    /* BIST activate (STP/SATA) */
        MCH_PMP_MASK            = 0xf,          /* PMP from cmd FIS (STP/SATA)*/

        CCTL_RST                = (1U << 5),    /* port logic reset */

                                                /* 0(LSB first), 1(MSB first) */
        CCTL_ENDIAN_DATA        = (1U << 3),    /* PRD data */
        CCTL_ENDIAN_RSP         = (1U << 2),    /* response frame */
        CCTL_ENDIAN_OPEN        = (1U << 1),    /* open address frame */
        CCTL_ENDIAN_CMD         = (1U << 0),    /* command table */

        /* MVS_Px_SER_CTLSTAT (per-phy control) */
        PHY_SSP_RST             = (1U << 3),    /* reset SSP link layer */
        PHY_BCAST_CHG           = (1U << 2),    /* broadcast(change) notif */
        PHY_RST_HARD            = (1U << 1),    /* hard reset + phy reset */
        PHY_RST                 = (1U << 0),    /* phy reset */

        /* MVS_Px_INT_STAT, MVS_Px_INT_MASK (per-phy events) */
        PHYEV_UNASSOC_FIS       = (1U << 19),   /* unassociated FIS rx'd */
        PHYEV_AN                = (1U << 18),   /* SATA async notification */
        PHYEV_BIST_ACT          = (1U << 17),   /* BIST activate FIS */
        PHYEV_SIG_FIS           = (1U << 16),   /* signature FIS */
        PHYEV_POOF              = (1U << 12),   /* phy ready from 1 -> 0 */
        PHYEV_IU_BIG            = (1U << 11),   /* IU too long err */
        PHYEV_IU_SMALL          = (1U << 10),   /* IU too short err */
        PHYEV_UNK_TAG           = (1U << 9),    /* unknown tag */
        PHYEV_BROAD_CH          = (1U << 8),    /* broadcast(CHANGE) */
        PHYEV_COMWAKE           = (1U << 7),    /* COMWAKE rx'd */
        PHYEV_PORT_SEL          = (1U << 6),    /* port selector present */
        PHYEV_HARD_RST          = (1U << 5),    /* hard reset rx'd */
        PHYEV_ID_TMOUT          = (1U << 4),    /* identify timeout */
        PHYEV_ID_FAIL           = (1U << 3),    /* identify failed */
        PHYEV_ID_DONE           = (1U << 2),    /* identify done */
        PHYEV_HARD_RST_DONE     = (1U << 1),    /* hard reset done */
        PHYEV_RDY_CH            = (1U << 0),    /* phy ready changed state */

        /* MVS_PCS */
        PCS_SATA_RETRY          = (1U << 8),    /* retry ctl FIS on R_ERR */
        PCS_RSP_RX_EN           = (1U << 7),    /* raw response rx */
        PCS_SELF_CLEAR          = (1U << 5),    /* self-clearing int mode */
        PCS_FIS_RX_EN           = (1U << 4),    /* FIS rx enable */
        PCS_CMD_STOP_ERR        = (1U << 3),    /* cmd stop-on-err enable */
        PCS_CMD_RST             = (1U << 2),    /* reset cmd issue */
        PCS_CMD_EN              = (1U << 0),    /* enable cmd issue */
};

enum mvs_info_flags {
        MVF_MSI                 = (1U << 0),    /* MSI is enabled */
        MVF_PHY_PWR_FIX         = (1U << 1),    /* bug workaround */
};

enum sas_cmd_port_registers {
        CMD_CMRST_OOB_DET       = 0x100, /* COMRESET OOB detect register */
        CMD_CMWK_OOB_DET        = 0x104, /* COMWAKE OOB detect register */
        CMD_CMSAS_OOB_DET       = 0x108, /* COMSAS OOB detect register */
        CMD_BRST_OOB_DET        = 0x10c, /* burst OOB detect register */
        CMD_OOB_SPACE           = 0x110, /* OOB space control register */
        CMD_OOB_BURST           = 0x114, /* OOB burst control register */
        CMD_PHY_TIMER           = 0x118, /* PHY timer control register */
        CMD_PHY_CONFIG0         = 0x11c, /* PHY config register 0 */
        CMD_PHY_CONFIG1         = 0x120, /* PHY config register 1 */
        CMD_SAS_CTL0            = 0x124, /* SAS control register 0 */
        CMD_SAS_CTL1            = 0x128, /* SAS control register 1 */
        CMD_SAS_CTL2            = 0x12c, /* SAS control register 2 */
        CMD_SAS_CTL3            = 0x130, /* SAS control register 3 */
        CMD_ID_TEST             = 0x134, /* ID test register */
        CMD_PL_TIMER            = 0x138, /* PL timer register */
        CMD_WD_TIMER            = 0x13c, /* WD timer register */
        CMD_PORT_SEL_COUNT      = 0x140, /* port selector count register */
        CMD_APP_MEM_CTL         = 0x144, /* Application Memory Control */
        CMD_XOR_MEM_CTL         = 0x148, /* XOR Block Memory Control */
        CMD_DMA_MEM_CTL         = 0x14c, /* DMA Block Memory Control */
        CMD_PORT_MEM_CTL0       = 0x150, /* Port Memory Control 0 */
        CMD_PORT_MEM_CTL1       = 0x154, /* Port Memory Control 1 */
        CMD_SATA_PORT_MEM_CTL0  = 0x158, /* SATA Port Memory Control 0 */
        CMD_SATA_PORT_MEM_CTL1  = 0x15c, /* SATA Port Memory Control 1 */
        CMD_XOR_MEM_BIST_CTL    = 0x160, /* XOR Memory BIST Control */
        CMD_XOR_MEM_BIST_STAT   = 0x164, /* XOR Memroy BIST Status */
        CMD_DMA_MEM_BIST_CTL    = 0x168, /* DMA Memory BIST Control */
        CMD_DMA_MEM_BIST_STAT   = 0x16c, /* DMA Memory BIST Status */
        CMD_PORT_MEM_BIST_CTL   = 0x170, /* Port Memory BIST Control */
        CMD_PORT_MEM_BIST_STAT0 = 0x174, /* Port Memory BIST Status 0 */
        CMD_PORT_MEM_BIST_STAT1 = 0x178, /* Port Memory BIST Status 1 */
        CMD_STP_MEM_BIST_CTL    = 0x17c, /* STP Memory BIST Control */
        CMD_STP_MEM_BIST_STAT0  = 0x180, /* STP Memory BIST Status 0 */
        CMD_STP_MEM_BIST_STAT1  = 0x184, /* STP Memory BIST Status 1 */
        CMD_RESET_COUNT         = 0x188, /* Reset Count */
        CMD_MONTR_DATA_SEL      = 0x18C, /* Monitor Data/Select */
        CMD_PLL_PHY_CONFIG      = 0x190, /* PLL/PHY Configuration */
        CMD_PHY_CTL             = 0x194, /* PHY Control and Status */
        CMD_PHY_TEST_COUNT0     = 0x198, /* Phy Test Count 0 */
        CMD_PHY_TEST_COUNT1     = 0x19C, /* Phy Test Count 1 */
        CMD_PHY_TEST_COUNT2     = 0x1A0, /* Phy Test Count 2 */
        CMD_APP_ERR_CONFIG      = 0x1A4, /* Application Error Configuration */
        CMD_PND_FIFO_CTL0       = 0x1A8, /* Pending FIFO Control 0 */
        CMD_HOST_CTL            = 0x1AC, /* Host Control Status */
        CMD_HOST_WR_DATA        = 0x1B0, /* Host Write Data */
        CMD_HOST_RD_DATA        = 0x1B4, /* Host Read Data */
        CMD_PHY_MODE_21         = 0x1B8, /* Phy Mode 21 */
        CMD_SL_MODE0            = 0x1BC, /* SL Mode 0 */
        CMD_SL_MODE1            = 0x1C0, /* SL Mode 1 */
        CMD_PND_FIFO_CTL1       = 0x1C4, /* Pending FIFO Control 1 */
};

/* SAS/SATA configuration port registers, aka phy registers */
enum sas_sata_config_port_regs {
        PHYR_IDENTIFY           = 0x0,  /* info for IDENTIFY frame */
        PHYR_ADDR_LO            = 0x4,  /* my SAS address (low) */
        PHYR_ADDR_HI            = 0x8,  /* my SAS address (high) */
        PHYR_ATT_DEV_INFO       = 0xC,  /* attached device info */
        PHYR_ATT_ADDR_LO        = 0x10, /* attached dev SAS addr (low) */
        PHYR_ATT_ADDR_HI        = 0x14, /* attached dev SAS addr (high) */
        PHYR_SATA_CTL           = 0x18, /* SATA control */
        PHYR_PHY_STAT           = 0x1C, /* PHY status */
        PHYR_WIDE_PORT          = 0x38, /* wide port participating */
        PHYR_CURRENT0           = 0x80, /* current connection info 0 */
        PHYR_CURRENT1           = 0x84, /* current connection info 1 */
        PHYR_CURRENT2           = 0x88, /* current connection info 2 */
};

enum pci_cfg_registers {
        PCR_PHY_CTL             = 0x40,
        PCR_PHY_CTL2            = 0x90,
};

enum pci_cfg_register_bits {
        PCTL_PWR_ON             = (0xFU << 24),
        PCTL_OFF                = (0xFU << 12),
};

enum nvram_layout_offsets {
        NVR_SIG                 = 0x00,         /* 0xAA, 0x55 */
        NVR_SAS_ADDR            = 0x02,         /* 8-byte SAS address */
};

enum chip_flavors {
        chip_6320,
        chip_6440,
        chip_6480,
};

struct mvs_chip_info {
        unsigned int            n_phy;
        unsigned int            srs_sz;
        unsigned int            slot_width;
};

struct mvs_err_info {
        __le32                  flags;
        __le32                  flags2;
};

struct mvs_prd {
        __le64                  addr;           /* 64-bit buffer address */
        __le32                  reserved;
        __le32                  len;            /* 16-bit length */
};

struct mvs_cmd_hdr {
        __le32                  flags;          /* PRD tbl len; SAS, SATA ctl */
        __le32                  lens;           /* cmd, max resp frame len */
        __le32                  tags;           /* targ port xfer tag; tag */
        __le32                  data_len;       /* data xfer len */
        __le64                  cmd_tbl;        /* command table address */
        __le64                  open_frame;     /* open addr frame address */
        __le64                  status_buf;     /* status buffer address */
        __le64                  prd_tbl;        /* PRD tbl address */
        __le32                  reserved[4];
};

struct mvs_slot_info {
        struct sas_task         *task;
        unsigned int            n_elem;

        /* DMA buffer for storing cmd tbl, open addr frame, status buffer,
         * and PRD table
         */
        void                    *buf;
        dma_addr_t              buf_dma;

        void                    *response;
};

struct mvs_port {
        struct asd_sas_port     sas_port;
};

struct mvs_phy {
        struct mvs_port         *port;
        struct asd_sas_phy      sas_phy;

        u8                      frame_rcvd[24 + 1024];
};

struct mvs_info {
        unsigned long           flags;

        spinlock_t              lock;           /* host-wide lock */
        struct pci_dev          *pdev;          /* our device */
        void __iomem            *regs;          /* enhanced mode registers */
        void __iomem            *peri_regs;     /* peripheral registers */

        u8                      sas_addr[SAS_ADDR_SIZE];
        struct sas_ha_struct    sas;            /* SCSI/SAS glue */
        struct Scsi_Host        *shost;

        __le32                  *tx;            /* TX (delivery) DMA ring */
        dma_addr_t              tx_dma;
        u32                     tx_prod;        /* cached next-producer idx */

        __le32                  *rx;            /* RX (completion) DMA ring */
        dma_addr_t              rx_dma;
        u32                     rx_cons;        /* RX consumer idx */

        __le32                  *rx_fis;        /* RX'd FIS area */
        dma_addr_t              rx_fis_dma;

        struct mvs_cmd_hdr      *slot;          /* DMA command header slots */
        dma_addr_t              slot_dma;

        const struct mvs_chip_info *chip;

                                        /* further per-slot information */
        struct mvs_slot_info    slot_info[MVS_SLOTS];
        unsigned long           tags[(MVS_SLOTS / sizeof(unsigned long)) + 1];

        struct mvs_phy          phy[MVS_MAX_PHYS];
        struct mvs_port         port[MVS_MAX_PHYS];
};

static struct scsi_transport_template *mvs_stt;

static const struct mvs_chip_info mvs_chips[] = {
        [chip_6320] =           { 2, 16, 9 },
        [chip_6440] =           { 4, 16, 9 },
        [chip_6480] =           { 8, 32, 10 },
};

static struct scsi_host_template mvs_sht = {
        .module                 = THIS_MODULE,
        .name                   = DRV_NAME,
        .queuecommand           = sas_queuecommand,
        .target_alloc           = sas_target_alloc,
        .slave_configure        = sas_slave_configure,
        .slave_destroy          = sas_slave_destroy,
        .change_queue_depth     = sas_change_queue_depth,
        .change_queue_type      = sas_change_queue_type,
        .bios_param             = sas_bios_param,
        .can_queue              = 1,
        .cmd_per_lun            = 1,
        .this_id                = -1,
        .sg_tablesize           = SG_ALL,
        .max_sectors            = SCSI_DEFAULT_MAX_SECTORS,
        .use_clustering         = ENABLE_CLUSTERING,
        .eh_device_reset_handler= sas_eh_device_reset_handler,
        .eh_bus_reset_handler   = sas_eh_bus_reset_handler,
        .slave_alloc            = sas_slave_alloc,
        .target_destroy         = sas_target_destroy,
        .ioctl                  = sas_ioctl,
};

static void mvs_int_rx(struct mvs_info *mvi, bool self_clear);

/* move to PCI layer or libata core? */
static int pci_go_64(struct pci_dev *pdev)
{
        int rc;

        if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
                rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
                if (rc) {
                        rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
                        if (rc) {
                                dev_printk(KERN_ERR, &pdev->dev,
                                           "64-bit DMA enable failed\n");
                                return rc;
                        }
                }
        } else {
                rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
                if (rc) {
                        dev_printk(KERN_ERR, &pdev->dev,
                                   "32-bit DMA enable failed\n");
                        return rc;
                }
                rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
                if (rc) {
                        dev_printk(KERN_ERR, &pdev->dev,
                                   "32-bit consistent DMA enable failed\n");
                        return rc;
                }
        }

        return rc;
}

static void mvs_tag_clear(struct mvs_info *mvi, unsigned int tag)
{
        mvi->tags[tag / sizeof(unsigned long)] &=
                ~(1UL << (tag % sizeof(unsigned long)));
}

static void mvs_tag_set(struct mvs_info *mvi, unsigned int tag)
{
        mvi->tags[tag / sizeof(unsigned long)] |=
                (1UL << (tag % sizeof(unsigned long)));
}

static bool mvs_tag_test(struct mvs_info *mvi, unsigned int tag)
{
        return mvi->tags[tag / sizeof(unsigned long)] &
                (1UL << (tag % sizeof(unsigned long)));
}

static int mvs_tag_alloc(struct mvs_info *mvi, unsigned int *tag_out)
{
        unsigned int i;

        for (i = 0; i < MVS_SLOTS; i++)
                if (!mvs_tag_test(mvi, i)) {
                        mvs_tag_set(mvi, i);
                        *tag_out = i;
                        return 0;
                }

        return -EBUSY;
}

static int mvs_eep_read(void __iomem *regs, unsigned int addr, u32 *data)
{
        int timeout = 1000;

        if (addr & ~SPI_ADDR_MASK)
                return -EINVAL;

        writel(addr, regs + SPI_CMD);
        writel(TWSI_RD, regs + SPI_CTL);

        while (timeout-- > 0) {
                if (readl(regs + SPI_CTL) & TWSI_RDY) {
                        *data = readl(regs + SPI_DATA);
                        return 0;
                }

                udelay(10);
        }

        return -EBUSY;
}

static int mvs_eep_read_buf(void __iomem *regs, unsigned int addr,
                            void *buf, unsigned int buflen)
{
        unsigned int addr_end, tmp_addr, i, j;
        u32 tmp = 0;
        int rc;
        u8 *tmp8, *buf8 = buf;

        addr_end = addr + buflen;
        tmp_addr = ALIGN(addr, 4);
        if (addr > 0xff)
                return -EINVAL;

        j = addr & 0x3;
        if (j) {
                rc = mvs_eep_read(regs, tmp_addr, &tmp);
                if (rc)
                        return rc;

                tmp8 = (u8 *) &tmp;
                for (i = j; i < 4; i++)
                        *buf8++ = tmp8[i];

                tmp_addr += 4;
        }

        for (j = ALIGN(addr_end, 4); tmp_addr < j; tmp_addr += 4) {
                rc = mvs_eep_read(regs, tmp_addr, &tmp);
                if (rc)
                        return rc;

                memcpy(buf8, &tmp, 4);
                buf8 += 4;
        }

        if (tmp_addr < addr_end) {
                rc = mvs_eep_read(regs, tmp_addr, &tmp);
                if (rc)
                        return rc;

                tmp8 = (u8 *) &tmp;
                j = addr_end - tmp_addr;
                for (i = 0; i < j; i++)
                        *buf8++ = tmp8[i];

                tmp_addr += 4;
        }

        return 0;
}

static int mvs_nvram_read(struct mvs_info *mvi, unsigned int addr,
                          void *buf, unsigned int buflen)
{
        void __iomem *regs = mvi->regs;
        int rc, i;
        unsigned int sum;
        u8 hdr[2], *tmp;
        const char *msg;

        rc = mvs_eep_read_buf(regs, addr, &hdr, 2);
        if (rc) {
                msg = "nvram hdr read failed";
                goto err_out;
        }
        rc = mvs_eep_read_buf(regs, addr + 2, buf, buflen);
        if (rc) {
                msg = "nvram read failed";
                goto err_out;
        }

        if (hdr[0] != 0x5A) {           /* entry id */
                msg = "invalid nvram entry id";
                rc = -ENOENT;
                goto err_out;
        }

        tmp = buf;
        sum = ((unsigned int)hdr[0]) + ((unsigned int)hdr[1]);
        for (i = 0; i < buflen; i++)
                sum += ((unsigned int)tmp[i]);

        if (sum) {
                msg = "nvram checksum failure";
                rc = -EILSEQ;
                goto err_out;
        }

        return 0;

err_out:
        dev_printk(KERN_ERR, &mvi->pdev->dev, "%s", msg);
        return rc;
}

static void mvs_int_port(struct mvs_info *mvi, int port_no, u32 events)
{
        /* FIXME */
}

static void mvs_int_sata(struct mvs_info *mvi)
{
        /* FIXME */
}

static void mvs_slot_free(struct mvs_info *mvi, struct sas_task *task,
                          struct mvs_slot_info *slot, unsigned int slot_idx)
{
        if (slot->n_elem)
                pci_unmap_sg(mvi->pdev, task->scatter,
                             slot->n_elem, task->data_dir);

        switch (task->task_proto) {
        case SAS_PROTOCOL_SMP:
                pci_unmap_sg(mvi->pdev, &task->smp_task.smp_resp, 1,
                             PCI_DMA_FROMDEVICE);
                pci_unmap_sg(mvi->pdev, &task->smp_task.smp_req, 1,
                             PCI_DMA_TODEVICE);
                break;

        case SAS_PROTOCOL_SATA:
        case SAS_PROTOCOL_STP:
        case SAS_PROTOCOL_SSP:
        default:
                /* do nothing */
                break;
        }

        mvs_tag_clear(mvi, slot_idx);
}

static void mvs_slot_err(struct mvs_info *mvi, struct sas_task *task,
                         unsigned int slot_idx)
{
        /* FIXME */
}

static void mvs_slot_complete(struct mvs_info *mvi, u32 rx_desc)
{
        unsigned int slot_idx = rx_desc & RXQ_SLOT_MASK;
        struct mvs_slot_info *slot = &mvi->slot_info[slot_idx];
        struct sas_task *task = slot->task;
        struct task_status_struct *tstat = &task->task_status;
        bool aborted;

        spin_lock(&task->task_state_lock);
        aborted = task->task_state_flags & SAS_TASK_STATE_ABORTED;
        if (!aborted) {
                task->task_state_flags &=
                        ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
                task->task_state_flags |= SAS_TASK_STATE_DONE;
        }
        spin_unlock(&task->task_state_lock);

        if (aborted)
                return;

        memset(tstat, 0, sizeof(*tstat));
        tstat->resp = SAS_TASK_COMPLETE;

        /* error info record present */
        if (rx_desc & RXQ_ERR) {
                tstat->stat = SAM_CHECK_COND;
                mvs_slot_err(mvi, task, slot_idx);
                goto out;
        }

        switch (task->task_proto) {
        case SAS_PROTOCOL_SSP:
                /* hw says status == 0, datapres == 0 */
                if (rx_desc & RXQ_GOOD)
                        tstat->stat = SAM_GOOD;

                /* response frame present */
                else if (rx_desc & RXQ_RSP) {
                        struct ssp_response_iu *iu =
                                slot->response + sizeof(struct mvs_err_info);
                        sas_ssp_task_response(&mvi->pdev->dev, task, iu);
                }

                /* should never happen? */
                else
                        tstat->stat = SAM_CHECK_COND;
                break;

        case SAS_PROTOCOL_SMP:
                tstat->stat = SAM_GOOD;
                break;

        case SAS_PROTOCOL_SATA:
        case SAS_PROTOCOL_STP:
                if ((rx_desc & (RXQ_DONE | RXQ_ERR | RXQ_ATTN)) == RXQ_DONE)
                        tstat->stat = SAM_GOOD;
                else
                        tstat->stat = SAM_CHECK_COND;
                /* FIXME: read taskfile data from SATA register set
                 * associated with SATA target
                 */
                break;

        default:
                tstat->stat = SAM_CHECK_COND;
                break;
        }

out:
        mvs_slot_free(mvi, task, slot, slot_idx);
        task->task_done(task);
}

static void mvs_int_full(struct mvs_info *mvi)
{
        void __iomem *regs = mvi->regs;
        u32 tmp, stat;
        int i;

        stat = mr32(INT_STAT);

        for (i = 0; i < MVS_MAX_PORTS; i++) {
                tmp = (stat >> i) & (CINT_PORT | CINT_PORT_STOPPED);
                if (tmp)
                        mvs_int_port(mvi, i, tmp);
        }

        if (stat & CINT_SRS)
                mvs_int_sata(mvi);

        if (stat & (CINT_CI_STOP | CINT_DONE))
                mvs_int_rx(mvi, false);

        mw32(INT_STAT, stat);
}

static void mvs_int_rx(struct mvs_info *mvi, bool self_clear)
{
        u32 rx_prod_idx, rx_desc;
        bool attn = false;

        /* the first dword in the RX ring is special: it contains
         * a mirror of the hardware's RX producer index, so that
         * we don't have to stall the CPU reading that register.
         * The actual RX ring is offset by one dword, due to this.
         */
        rx_prod_idx = le32_to_cpu(mvi->rx[0]) & 0xfff;
        if (rx_prod_idx == 0xfff) {     /* h/w hasn't touched RX ring yet */
                mvi->rx_cons = 0xfff;
                return;
        }
        if (mvi->rx_cons == 0xfff)
                mvi->rx_cons = MVS_RX_RING_SZ - 1;

        while (mvi->rx_cons != rx_prod_idx) {
                /* increment our internal RX consumer pointer */
                mvi->rx_cons = (mvi->rx_cons + 1) & (MVS_RX_RING_SZ - 1);

                /* Read RX descriptor at offset+1, due to above */
                rx_desc = le32_to_cpu(mvi->rx[mvi->rx_cons + 1]);

                if (rx_desc & RXQ_DONE)
                        /* we had a completion, error or no */
                        mvs_slot_complete(mvi, rx_desc);

                if (rx_desc & RXQ_ATTN)
                        attn = true;
        }

        if (attn && self_clear)
                mvs_int_full(mvi);

}

static irqreturn_t mvs_interrupt(int irq, void *opaque)
{
        struct mvs_info *mvi = opaque;
        void __iomem *regs = mvi->regs;
        u32 stat;

        stat = mr32(GBL_INT_STAT);
        if (stat == 0 || stat == 0xffffffff)
                return IRQ_NONE;

        spin_lock(&mvi->lock);

        mvs_int_full(mvi);

        spin_unlock(&mvi->lock);

        return IRQ_HANDLED;
}

static irqreturn_t mvs_msi_interrupt(int irq, void *opaque)
{
        struct mvs_info *mvi = opaque;

        spin_lock(&mvi->lock);

        mvs_int_rx(mvi, true);

        spin_unlock(&mvi->lock);

        return IRQ_HANDLED;
}

struct mvs_task_exec_info {
        struct sas_task         *task;
        struct mvs_cmd_hdr      *hdr;
        unsigned int            tag;
        int                     n_elem;
};

static int mvs_task_prep_smp(struct mvs_info *mvi, struct mvs_task_exec_info *tei)
{
        int elem, rc;
        struct mvs_cmd_hdr *hdr = tei->hdr;
        struct scatterlist *sg_req, *sg_resp;
        unsigned int req_len, resp_len, tag = tei->tag;

        /*
         * DMA-map SMP request, response buffers
         */

        sg_req = &tei->task->smp_task.smp_req;
        elem = pci_map_sg(mvi->pdev, sg_req, 1, PCI_DMA_TODEVICE);
        if (!elem)
                return -ENOMEM;
        req_len = sg_dma_len(sg_req);

        sg_resp = &tei->task->smp_task.smp_resp;
        elem = pci_map_sg(mvi->pdev, sg_resp, 1, PCI_DMA_FROMDEVICE);
        if (!elem) {
                rc = -ENOMEM;
                goto err_out;
        }
        resp_len = sg_dma_len(sg_resp);

        /* must be in dwords */
        if ((req_len & 0x3) || (resp_len & 0x3)) {
                rc = -EINVAL;
                goto err_out_2;
        }

        /*
         * Fill in TX ring and command slot header
         */

        mvi->tx[tag] = cpu_to_le32(
                (TXQ_CMD_SMP << TXQ_CMD_SHIFT) | TXQ_MODE_I | tag);

        hdr->flags = 0;
        hdr->lens = cpu_to_le32(((resp_len / 4) << 16) | (req_len / 4));
        hdr->tags = cpu_to_le32(tag);
        hdr->data_len = 0;
        hdr->cmd_tbl = cpu_to_le64(sg_dma_address(sg_req));
        hdr->open_frame = 0;
        hdr->status_buf = cpu_to_le64(sg_dma_address(sg_resp));
        hdr->prd_tbl = 0;

        return 0;

err_out_2:
        pci_unmap_sg(mvi->pdev, &tei->task->smp_task.smp_resp, 1,
                     PCI_DMA_FROMDEVICE);
err_out:
        pci_unmap_sg(mvi->pdev, &tei->task->smp_task.smp_req, 1,
                     PCI_DMA_TODEVICE);
        return rc;
}

static int mvs_task_prep_ata(struct mvs_info *mvi,
                             struct mvs_task_exec_info *tei)
{
        struct sas_task *task = tei->task;
        struct domain_device *dev = task->dev;
        struct mvs_cmd_hdr *hdr = tei->hdr;
        struct asd_sas_port *sas_port = dev->port;
        unsigned int tag = tei->tag;
        struct mvs_slot_info *slot = &mvi->slot_info[tag];
        u32 flags = (tei->n_elem << MCH_PRD_LEN_SHIFT);
        struct scatterlist *sg;
        struct mvs_prd *buf_prd;
        void *buf_tmp;
        u8 *buf_cmd, *buf_oaf;
        dma_addr_t buf_tmp_dma;
        unsigned int i, req_len, resp_len;

        /* FIXME: fill in SATA register set */
        mvi->tx[tag] = cpu_to_le32(TXQ_MODE_I | tag |
                (TXQ_CMD_STP << TXQ_CMD_SHIFT) |
                (sas_port->phy_mask << TXQ_PHY_SHIFT));

        if (task->ata_task.use_ncq)
                flags |= MCH_FPDMA;
        if (dev->sata_dev.command_set == ATAPI_COMMAND_SET)
                flags |= MCH_ATAPI;
        /* FIXME: fill in port multiplier number */

        hdr->flags = cpu_to_le32(flags);
        hdr->tags = cpu_to_le32(tag);
        hdr->data_len = cpu_to_le32(task->total_xfer_len);

        /*
         * arrange MVS_SLOT_BUF_SZ-sized DMA buffer according to our needs
         */
        memset(slot->buf, 0, MVS_SLOT_BUF_SZ);

        /* region 1: command table area (MVS_ATA_CMD_SZ bytes) ***************/
        buf_cmd =
        buf_tmp = slot->buf;
        buf_tmp_dma = slot->buf_dma;

        hdr->cmd_tbl = cpu_to_le64(buf_tmp_dma);

        buf_tmp += MVS_ATA_CMD_SZ;
        buf_tmp_dma += MVS_ATA_CMD_SZ;

        /* region 2: open address frame area (MVS_OAF_SZ bytes) **********/
        /* used for STP.  unused for SATA? */
        buf_oaf = buf_tmp;
        hdr->open_frame = cpu_to_le64(buf_tmp_dma);

        buf_tmp += MVS_OAF_SZ;
        buf_tmp_dma += MVS_OAF_SZ;

        /* region 3: PRD table ***********************************************/
        buf_prd = buf_tmp;
        hdr->prd_tbl = cpu_to_le64(buf_tmp_dma);

        i = sizeof(struct mvs_prd) * tei->n_elem;
        buf_tmp += i;
        buf_tmp_dma += i;

        /* region 4: status buffer (larger the PRD, smaller this buf) ********/
        /* FIXME: probably unused, for SATA.  kept here just in case
         * we get a STP/SATA error information record
         */
        slot->response = buf_tmp;
        hdr->status_buf = cpu_to_le64(buf_tmp_dma);

        req_len = sizeof(struct ssp_frame_hdr) + 28;
        resp_len = MVS_SLOT_BUF_SZ - MVS_ATA_CMD_SZ -
                   sizeof(struct mvs_err_info) - i;

        /* request, response lengths */
        hdr->lens = cpu_to_le32(((resp_len / 4) << 16) | (req_len / 4));

        /* fill in command FIS and ATAPI CDB */
        memcpy(buf_cmd, &task->ata_task.fis,
               sizeof(struct host_to_dev_fis));
        memcpy(buf_cmd + 0x40, task->ata_task.atapi_packet, 16);

        /* fill in PRD (scatter/gather) table, if any */
        sg = task->scatter;
        for (i = 0; i < tei->n_elem; i++) {
                buf_prd->addr = cpu_to_le64(sg_dma_address(sg));
                buf_prd->len = cpu_to_le32(sg_dma_len(sg));

                sg++;
                buf_prd++;
        }

        return 0;
}

static int mvs_task_prep_ssp(struct mvs_info *mvi,
                             struct mvs_task_exec_info *tei)
{
        struct sas_task *task = tei->task;
        struct asd_sas_port *sas_port = task->dev->port;
        struct mvs_cmd_hdr *hdr = tei->hdr;
        struct mvs_slot_info *slot;
        struct scatterlist *sg;
        unsigned int resp_len, req_len, i, tag = tei->tag;
        struct mvs_prd *buf_prd;
        struct ssp_frame_hdr *ssp_hdr;
        void *buf_tmp;
        u8 *buf_cmd, *buf_oaf, fburst = 0;
        dma_addr_t buf_tmp_dma;
        u32 flags;

        slot = &mvi->slot_info[tag];

        mvi->tx[tag] = cpu_to_le32(TXQ_MODE_I | tag |
                (TXQ_CMD_SSP << TXQ_CMD_SHIFT) |
                (sas_port->phy_mask << TXQ_PHY_SHIFT));

        flags = MCH_RETRY;
        if (task->ssp_task.enable_first_burst) {
                flags |= MCH_FBURST;
                fburst = (1 << 7);
        }
        hdr->flags = cpu_to_le32(flags |
                (tei->n_elem << MCH_PRD_LEN_SHIFT) |
                (MCH_SSP_FR_CMD << MCH_SSP_FR_TYPE_SHIFT));

        hdr->tags = cpu_to_le32(tag);
        hdr->data_len = cpu_to_le32(task->total_xfer_len);

        /*
         * arrange MVS_SLOT_BUF_SZ-sized DMA buffer according to our needs
         */
        memset(slot->buf, 0, MVS_SLOT_BUF_SZ);

        /* region 1: command table area (MVS_SSP_CMD_SZ bytes) ***************/
        buf_cmd =
        buf_tmp = slot->buf;
        buf_tmp_dma = slot->buf_dma;

        hdr->cmd_tbl = cpu_to_le64(buf_tmp_dma);

        buf_tmp += MVS_SSP_CMD_SZ;
        buf_tmp_dma += MVS_SSP_CMD_SZ;

        /* region 2: open address frame area (MVS_OAF_SZ bytes) **********/
        buf_oaf = buf_tmp;
        hdr->open_frame = cpu_to_le64(buf_tmp_dma);

        buf_tmp += MVS_OAF_SZ;
        buf_tmp_dma += MVS_OAF_SZ;

        /* region 3: PRD table ***********************************************/
        buf_prd = buf_tmp;
        hdr->prd_tbl = cpu_to_le64(buf_tmp_dma);

        i = sizeof(struct mvs_prd) * tei->n_elem;
        buf_tmp += i;
        buf_tmp_dma += i;

        /* region 4: status buffer (larger the PRD, smaller this buf) ********/
        slot->response = buf_tmp;
        hdr->status_buf = cpu_to_le64(buf_tmp_dma);

        req_len = sizeof(struct ssp_frame_hdr) + 28;
        resp_len = MVS_SLOT_BUF_SZ - MVS_SSP_CMD_SZ - MVS_OAF_SZ -
                   sizeof(struct mvs_err_info) - i;

        /* request, response lengths */
        hdr->lens = cpu_to_le32(((resp_len / 4) << 16) | (req_len / 4));

        /* generate open address frame hdr (first 12 bytes) */
        buf_oaf[0] = (1 << 7) | (1 << 4) | 0x1; /* initiator, SSP, ftype 1h */
        buf_oaf[1] = task->dev->linkrate & 0xf;
        buf_oaf[2] = tag >> 8;
        buf_oaf[3] = tag;
        memcpy(buf_oaf + 4, task->dev->sas_addr, SAS_ADDR_SIZE);

        /* fill in SSP frame header */
        ssp_hdr = (struct ssp_frame_hdr *) buf_cmd;
        ssp_hdr->frame_type = SSP_COMMAND;
        memcpy(ssp_hdr->hashed_dest_addr, task->dev->hashed_sas_addr,
               HASHED_SAS_ADDR_SIZE);
        memcpy(ssp_hdr->hashed_src_addr,
               task->dev->port->ha->hashed_sas_addr, HASHED_SAS_ADDR_SIZE);
        ssp_hdr->tag = cpu_to_be16(tag);

        /* fill in command frame IU */
        buf_cmd += sizeof(*ssp_hdr);
        memcpy(buf_cmd, &task->ssp_task.LUN, 8);
        buf_cmd[9] = fburst |
                task->ssp_task.task_attr |
                (task->ssp_task.task_prio << 3);
        memcpy(buf_cmd + 12, &task->ssp_task.cdb, 16);

        /* fill in PRD (scatter/gather) table, if any */
        sg = task->scatter;
        for (i = 0; i < tei->n_elem; i++) {
                buf_prd->addr = cpu_to_le64(sg_dma_address(sg));
                buf_prd->len = cpu_to_le32(sg_dma_len(sg));

                sg++;
                buf_prd++;
        }

        return 0;
}

static int mvs_task_exec(struct sas_task *task, const int num, gfp_t gfp_flags)
{
        struct mvs_info *mvi = task->dev->port->ha->lldd_ha;
        unsigned int tag = 0xdeadbeef, rc, n_elem = 0;
        void __iomem *regs = mvi->regs;
        unsigned long flags;
        struct mvs_task_exec_info tei;

        /* FIXME: STP/SATA support not complete yet */
        if (task->task_proto == SAS_PROTOCOL_SATA || task->task_proto == SAS_PROTOCOL_STP)
                return -SAS_DEV_NO_RESPONSE;

        if (task->num_scatter) {
                n_elem = pci_map_sg(mvi->pdev, task->scatter,
                                    task->num_scatter, task->data_dir);
                if (!n_elem)
                        return -ENOMEM;
        }

        spin_lock_irqsave(&mvi->lock, flags);

        rc = mvs_tag_alloc(mvi, &tag);
        if (rc)
                goto err_out;

        mvi->slot_info[tag].task = task;
        mvi->slot_info[tag].n_elem = n_elem;
        tei.task = task;
        tei.hdr = &mvi->slot[tag];
        tei.tag = tag;
        tei.n_elem = n_elem;

        switch (task->task_proto) {
        case SAS_PROTOCOL_SMP:
                rc = mvs_task_prep_smp(mvi, &tei);
                break;
        case SAS_PROTOCOL_SSP:
                rc = mvs_task_prep_ssp(mvi, &tei);
                break;
        case SAS_PROTOCOL_SATA:
        case SAS_PROTOCOL_STP:
                rc = mvs_task_prep_ata(mvi, &tei);
                break;
        default:
                rc = -EINVAL;
                break;
        }

        if (rc)
                goto err_out_tag;

        /* TODO: select normal or high priority */

        mw32(RX_PROD_IDX, mvi->tx_prod);

        mvi->tx_prod = (mvi->tx_prod + 1) & (MVS_TX_RING_SZ - 1);

        spin_lock(&task->task_state_lock);
        task->task_state_flags |= SAS_TASK_AT_INITIATOR;
        spin_unlock(&task->task_state_lock);

        spin_unlock_irqrestore(&mvi->lock, flags);
        return 0;

err_out_tag:
        mvs_tag_clear(mvi, tag);
err_out:
        if (n_elem)
                pci_unmap_sg(mvi->pdev, task->scatter, n_elem, task->data_dir);
        spin_unlock_irqrestore(&mvi->lock, flags);
        return rc;
}

static void mvs_free(struct mvs_info *mvi)
{
        int i;

        if (!mvi)
                return;

        for (i = 0; i < MVS_SLOTS; i++) {
                struct mvs_slot_info *slot = &mvi->slot_info[i];

                if (slot->buf)
                        dma_free_coherent(&mvi->pdev->dev, MVS_SLOT_BUF_SZ,
                                          slot->buf, slot->buf_dma);
        }

        if (mvi->tx)
                dma_free_coherent(&mvi->pdev->dev,
                                  sizeof(*mvi->tx) * MVS_TX_RING_SZ,
                                  mvi->tx, mvi->tx_dma);
        if (mvi->rx_fis)
                dma_free_coherent(&mvi->pdev->dev, MVS_RX_FISL_SZ,
                                  mvi->rx_fis, mvi->rx_fis_dma);
        if (mvi->rx)
                dma_free_coherent(&mvi->pdev->dev,
                                  sizeof(*mvi->rx) * MVS_RX_RING_SZ,
                                  mvi->rx, mvi->rx_dma);
        if (mvi->slot)
                dma_free_coherent(&mvi->pdev->dev,
                                  sizeof(*mvi->slot) * MVS_RX_RING_SZ,
                                  mvi->slot, mvi->slot_dma);
        if (mvi->peri_regs)
                iounmap(mvi->peri_regs);
        if (mvi->regs)
                iounmap(mvi->regs);
        if (mvi->shost)
                scsi_host_put(mvi->shost);
        kfree(mvi->sas.sas_port);
        kfree(mvi->sas.sas_phy);
        kfree(mvi);
}

/* FIXME: locking? */
static int mvs_phy_control(struct asd_sas_phy *sas_phy, enum phy_func func,
                           void *funcdata)
{
        struct mvs_info *mvi = sas_phy->ha->lldd_ha;
        void __iomem *reg;
        int rc = 0, phy_id = sas_phy->id;
        u32 tmp;

        reg = mvi->regs + MVS_P0_SER_CTLSTAT + (phy_id * 4);

        switch (func) {
        case PHY_FUNC_SET_LINK_RATE: {
                struct sas_phy_linkrates *rates = funcdata;
                u32 lrmin = 0, lrmax = 0;

                lrmin = (rates->minimum_linkrate << 8);
                lrmax = (rates->maximum_linkrate << 12);

                tmp = readl(reg);
                if (lrmin) {
                        tmp &= ~(0xf << 8);
                        tmp |= lrmin;
                }
                if (lrmax) {
                        tmp &= ~(0xf << 12);
                        tmp |= lrmax;
                }
                writel(tmp, reg);
                break;
        }

        case PHY_FUNC_HARD_RESET:
                tmp = readl(reg);
                if (tmp & PHY_RST_HARD)
                        break;
                writel(tmp | PHY_RST_HARD, reg);
                break;

        case PHY_FUNC_LINK_RESET:
                writel(readl(reg) | PHY_RST, reg);
                break;

        case PHY_FUNC_DISABLE:
        case PHY_FUNC_RELEASE_SPINUP_HOLD:
        default:
                rc = -EOPNOTSUPP;
        }

        return rc;
}

static void __devinit mvs_phy_init(struct mvs_info *mvi, int phy_id)
{
        struct mvs_phy *phy = &mvi->phy[phy_id];
        struct asd_sas_phy *sas_phy = &phy->sas_phy;

        sas_phy->enabled = (phy_id < mvi->chip->n_phy) ? 1 : 0;
        sas_phy->class = SAS;
        sas_phy->iproto = SAS_PROTOCOL_ALL;
        sas_phy->tproto = 0;
        sas_phy->type = PHY_TYPE_PHYSICAL;
        sas_phy->role = PHY_ROLE_INITIATOR;
        sas_phy->oob_mode = OOB_NOT_CONNECTED;
        sas_phy->linkrate = SAS_LINK_RATE_UNKNOWN;

        sas_phy->id = phy_id;
        sas_phy->sas_addr = &mvi->sas_addr[0];
        sas_phy->frame_rcvd = &phy->frame_rcvd[0];
        sas_phy->ha = &mvi->sas;
        sas_phy->lldd_phy = phy;
}

static struct mvs_info * __devinit mvs_alloc(struct pci_dev *pdev,
                                const struct pci_device_id *ent)
{
        struct mvs_info *mvi;
        unsigned long res_start, res_len;
        struct asd_sas_phy **arr_phy;
        struct asd_sas_port **arr_port;
        const struct mvs_chip_info *chip = &mvs_chips[ent->driver_data];
        int i;

        /*
         * alloc and init our per-HBA mvs_info struct
         */

        mvi = kzalloc(sizeof(*mvi), GFP_KERNEL);
        if (!mvi)
                return NULL;

        spin_lock_init(&mvi->lock);
        mvi->pdev = pdev;
        mvi->chip = chip;

        if (pdev->device == 0x6440 && pdev->revision == 0)
                mvi->flags |= MVF_PHY_PWR_FIX;

        /*
         * alloc and init SCSI, SAS glue
         */

        mvi->shost = scsi_host_alloc(&mvs_sht, sizeof(void *));
        if (!mvi->shost)
                goto err_out;

        arr_phy = kcalloc(MVS_MAX_PHYS, sizeof(void *), GFP_KERNEL);
        arr_port = kcalloc(MVS_MAX_PHYS, sizeof(void *), GFP_KERNEL);
        if (!arr_phy || !arr_port)
                goto err_out;

        for (i = 0; i < MVS_MAX_PHYS; i++) {
                mvs_phy_init(mvi, i);
                arr_phy[i] = &mvi->phy[i].sas_phy;
                arr_port[i] = &mvi->port[i].sas_port;
        }

        SHOST_TO_SAS_HA(mvi->shost) = &mvi->sas;
        mvi->shost->transportt = mvs_stt;
        mvi->shost->max_id = ~0;
        mvi->shost->max_lun = ~0;
        mvi->shost->max_cmd_len = ~0;

        mvi->sas.sas_ha_name = DRV_NAME;
        mvi->sas.dev = &pdev->dev;
        mvi->sas.lldd_module = THIS_MODULE;
        mvi->sas.sas_addr = &mvi->sas_addr[0];
        mvi->sas.sas_phy = arr_phy;
        mvi->sas.sas_port = arr_port;
        mvi->sas.num_phys = chip->n_phy;
        mvi->sas.lldd_max_execute_num = MVS_TX_RING_SZ - 1;/* FIXME: correct? */
        mvi->sas.lldd_queue_size = MVS_TX_RING_SZ - 1;     /* FIXME: correct? */
        mvi->sas.lldd_ha = mvi;
        mvi->sas.core.shost = mvi->shost;

        mvs_tag_set(mvi, MVS_TX_RING_SZ - 1);

        /*
         * ioremap main and peripheral registers
         */

        res_start = pci_resource_start(pdev, 2);
        res_len = pci_resource_len(pdev, 2);
        if (!res_start || !res_len)
                goto err_out;

        mvi->peri_regs = ioremap_nocache(res_start, res_len);
        if (!mvi->regs)
                goto err_out;

        res_start = pci_resource_start(pdev, 4);
        res_len = pci_resource_len(pdev, 4);
        if (!res_start || !res_len)
                goto err_out;

        mvi->regs = ioremap_nocache(res_start, res_len);
        if (!mvi->regs)
                goto err_out;

        /*
         * alloc and init our DMA areas
         */

        mvi->tx = dma_alloc_coherent(&pdev->dev,
                                     sizeof(*mvi->tx) * MVS_TX_RING_SZ,
                                     &mvi->tx_dma, GFP_KERNEL);
        if (!mvi->tx)
                goto err_out;
        memset(mvi->tx, 0, sizeof(*mvi->tx) * MVS_TX_RING_SZ);

        mvi->rx_fis = dma_alloc_coherent(&pdev->dev, MVS_RX_FISL_SZ,
                                     &mvi->rx_fis_dma, GFP_KERNEL);
        if (!mvi->rx_fis)
                goto err_out;
        memset(mvi->rx_fis, 0, MVS_RX_FISL_SZ);

        mvi->rx = dma_alloc_coherent(&pdev->dev,
                                     sizeof(*mvi->rx) * MVS_RX_RING_SZ,
                                     &mvi->rx_dma, GFP_KERNEL);
        if (!mvi->rx)
                goto err_out;
        memset(mvi->rx, 0, sizeof(*mvi->rx) * MVS_RX_RING_SZ);

        mvi->rx[0] = cpu_to_le32(0xfff);
        mvi->rx_cons = 0xfff;

        mvi->slot = dma_alloc_coherent(&pdev->dev,
                                       sizeof(*mvi->slot) * MVS_SLOTS,
                                       &mvi->slot_dma, GFP_KERNEL);
        if (!mvi->slot)
                goto err_out;
        memset(mvi->slot, 0, sizeof(*mvi->slot) * MVS_SLOTS);

        for (i = 0; i < MVS_SLOTS; i++) {
                struct mvs_slot_info *slot = &mvi->slot_info[i];

                slot->buf = dma_alloc_coherent(&pdev->dev, MVS_SLOT_BUF_SZ,
                                       &slot->buf_dma, GFP_KERNEL);
                if (!slot->buf)
                        goto err_out;
                memset(slot->buf, 0, MVS_SLOT_BUF_SZ);
        }

        /* finally, read NVRAM to get our SAS address */
        if (mvs_nvram_read(mvi, NVR_SAS_ADDR, &mvi->sas_addr, 8))
                goto err_out;

        return mvi;

err_out:
        mvs_free(mvi);
        return NULL;
}

static u32 mvs_cr32(void __iomem *regs, u32 addr)
{
        mw32(CMD_ADDR, addr);
        return mr32(CMD_DATA);
}

static void mvs_cw32(void __iomem *regs, u32 addr, u32 val)
{
        mw32(CMD_ADDR, addr);
        mw32(CMD_DATA, val);
}

#if 0
static u32 mvs_phy_read(struct mvs_info *mvi, unsigned int phy_id, u32 addr)
{
        void __iomem *regs = mvi->regs;
        void __iomem *phy_regs = regs + MVS_P0_CFG_ADDR + (phy_id * 8);

        writel(addr, phy_regs);
        return readl(phy_regs + 4);
}
#endif

static void mvs_phy_write(struct mvs_info *mvi, unsigned int phy_id,
                          u32 addr, u32 val)
{
        void __iomem *regs = mvi->regs;
        void __iomem *phy_regs = regs + MVS_P0_CFG_ADDR + (phy_id * 8);

        writel(addr, phy_regs);
        writel(val, phy_regs + 4);
        readl(phy_regs);        /* flush */
}

static void __devinit mvs_phy_hacks(struct mvs_info *mvi)
{
        void __iomem *regs = mvi->regs;
        u32 tmp;

        /* workaround for SATA R-ERR, to ignore phy glitch */
        tmp = mvs_cr32(regs, CMD_PHY_TIMER);
        tmp &= ~(1 << 9);
        tmp |= (1 << 10);
        mvs_cw32(regs, CMD_PHY_TIMER, tmp);

        /* enable retry 127 times */
        mvs_cw32(regs, CMD_SAS_CTL1, 0x7f7f);

        /* extend open frame timeout to max */
        tmp = mvs_cr32(regs, CMD_SAS_CTL0);
        tmp &= ~0xffff;
        tmp |= 0x3fff;
        mvs_cw32(regs, CMD_SAS_CTL0, tmp);

        /* workaround for WDTIMEOUT , set to 550 ms */
        mvs_cw32(regs, CMD_WD_TIMER, 0xffffff);

        /* not to halt for different port op during wideport link change */
        mvs_cw32(regs, CMD_APP_ERR_CONFIG, 0xffefbf7d);

        /* workaround for Seagate disk not-found OOB sequence, recv
         * COMINIT before sending out COMWAKE */
        tmp = mvs_cr32(regs, CMD_PHY_MODE_21);
        tmp &= 0x0000ffff;
        tmp |= 0x00fa0000;
        mvs_cw32(regs, CMD_PHY_MODE_21, tmp);

        tmp = mvs_cr32(regs, CMD_PHY_TIMER);
        tmp &= 0x1fffffff;
        tmp |= (2U << 29);      /* 8 ms retry */
        mvs_cw32(regs, CMD_PHY_TIMER, tmp);
}

static int __devinit mvs_hw_init(struct mvs_info *mvi)
{
        void __iomem *regs = mvi->regs;
        int i;
        u32 tmp, cctl;

        /* make sure interrupts are masked immediately (paranoia) */
        mw32(GBL_CTL, 0);
        tmp = mr32(GBL_CTL);

        if (!(tmp & HBA_RST)) {
                if (mvi->flags & MVF_PHY_PWR_FIX) {
                        pci_read_config_dword(mvi->pdev, PCR_PHY_CTL, &tmp);
                        tmp &= ~PCTL_PWR_ON;
                        tmp |= PCTL_OFF;
                        pci_write_config_dword(mvi->pdev, PCR_PHY_CTL, tmp);

                        pci_read_config_dword(mvi->pdev, PCR_PHY_CTL2, &tmp);
                        tmp &= ~PCTL_PWR_ON;
                        tmp |= PCTL_OFF;
                        pci_write_config_dword(mvi->pdev, PCR_PHY_CTL2, tmp);
                }

                /* global reset, incl. COMRESET/H_RESET_N (self-clearing) */
                mw32_f(GBL_CTL, HBA_RST);
        }


        /* wait for reset to finish; timeout is just a guess */
        i = 1000;
        while (i-- > 0) {
                msleep(10);

                if (!(mr32(GBL_CTL) & HBA_RST))
                        break;
        }
        if (mr32(GBL_CTL) & HBA_RST) {
                dev_printk(KERN_ERR, &mvi->pdev->dev, "HBA reset failed\n");
                return -EBUSY;
        }

        /* make sure RST is set; HBA_RST /should/ have done that for us */
        cctl = mr32(CTL);
        if (cctl & CCTL_RST)
                cctl &= ~CCTL_RST;
        else
                mw32_f(CTL, cctl | CCTL_RST);

        pci_read_config_dword(mvi->pdev, PCR_PHY_CTL, &tmp);
        tmp |= PCTL_PWR_ON;
        tmp &= ~PCTL_OFF;
        pci_write_config_dword(mvi->pdev, PCR_PHY_CTL, tmp);

        pci_read_config_dword(mvi->pdev, PCR_PHY_CTL2, &tmp);
        tmp |= PCTL_PWR_ON;
        tmp &= ~PCTL_OFF;
        pci_write_config_dword(mvi->pdev, PCR_PHY_CTL2, tmp);

        mw32_f(CTL, cctl);

        mvs_phy_hacks(mvi);

        mw32(CMD_LIST_LO, mvi->slot_dma);
        mw32(CMD_LIST_HI, (mvi->slot_dma >> 16) >> 16);

        mw32(RX_FIS_LO, mvi->rx_fis_dma);
        mw32(RX_FIS_HI, (mvi->rx_fis_dma >> 16) >> 16);

        mw32(TX_CFG, MVS_TX_RING_SZ);
        mw32(TX_LO, mvi->tx_dma);
        mw32(TX_HI, (mvi->tx_dma >> 16) >> 16);

        mw32(RX_CFG, MVS_RX_RING_SZ);
        mw32(RX_LO, mvi->rx_dma);
        mw32(RX_HI, (mvi->rx_dma >> 16) >> 16);

        /* init and reset phys */
        for (i = 0; i < mvi->chip->n_phy; i++) {
                /* FIXME: is this the correct dword order? */
                u32 lo = *((u32 *) &mvi->sas_addr[0]);
                u32 hi = *((u32 *) &mvi->sas_addr[4]);

                /* set phy local SAS address */
                mvs_phy_write(mvi, i, PHYR_ADDR_LO, lo);
                mvs_phy_write(mvi, i, PHYR_ADDR_HI, hi);

                /* reset phy */
                tmp = readl(regs + MVS_P0_SER_CTLSTAT + (i * 4));
                tmp |= PHY_RST;
                writel(tmp, regs + MVS_P0_SER_CTLSTAT + (i * 4));
        }

        msleep(100);

        for (i = 0; i < mvi->chip->n_phy; i++) {
                /* set phy int mask */
                writel(PHYEV_BROAD_CH | PHYEV_RDY_CH,
                       regs + MVS_P0_INT_MASK + (i * 8));

                /* clear phy int status */
                tmp = readl(regs + MVS_P0_INT_STAT + (i * 8));
                writel(tmp, regs + MVS_P0_INT_STAT + (i * 8));
        }

        /* FIXME: update wide port bitmaps */

        /* ladies and gentlemen, start your engines */
        mw32(TX_CFG, MVS_TX_RING_SZ | TX_EN);
        mw32(RX_CFG, MVS_RX_RING_SZ | RX_EN);
        mw32(PCS, PCS_SATA_RETRY | PCS_FIS_RX_EN | PCS_CMD_EN |
             ((mvi->flags & MVF_MSI) ? PCS_SELF_CLEAR : 0));

        /* re-enable interrupts globally */
        mw32(GBL_CTL, INT_EN);

        return 0;
}

static void __devinit mvs_print_info(struct mvs_info *mvi)
{
        struct pci_dev *pdev = mvi->pdev;
        static int printed_version;

        if (!printed_version++)
                dev_printk(KERN_INFO, &pdev->dev, "version " DRV_VERSION "\n");

        dev_printk(KERN_INFO, &pdev->dev, "%u phys, addr %llx\n",
                   mvi->chip->n_phy, SAS_ADDR(mvi->sas_addr));
}

static int __devinit mvs_pci_init(struct pci_dev *pdev,
                                 const struct pci_device_id *ent)
{
        int rc;
        struct mvs_info *mvi;
        irq_handler_t irq_handler = mvs_interrupt;

        rc = pci_enable_device(pdev);
        if (rc)
                return rc;

        pci_set_master(pdev);

        rc = pci_request_regions(pdev, DRV_NAME);
        if (rc)
                goto err_out_disable;

        rc = pci_go_64(pdev);
        if (rc)
                goto err_out_regions;

        mvi = mvs_alloc(pdev, ent);
        if (!mvi) {
                rc = -ENOMEM;
                goto err_out_regions;
        }

        rc = mvs_hw_init(mvi);
        if (rc)
                goto err_out_mvi;

        if (!pci_enable_msi(pdev)) {
                mvi->flags |= MVF_MSI;
                irq_handler = mvs_msi_interrupt;
        }

        rc = request_irq(pdev->irq, irq_handler, IRQF_SHARED, DRV_NAME, mvi);
        if (rc)
                goto err_out_msi;

        rc = scsi_add_host(mvi->shost, &pdev->dev);
        if (rc)
                goto err_out_irq;

        rc = sas_register_ha(&mvi->sas);
        if (rc)
                goto err_out_shost;

        pci_set_drvdata(pdev, mvi);

        mvs_print_info(mvi);

        scsi_scan_host(mvi->shost);
        return 0;

err_out_shost:
        scsi_remove_host(mvi->shost);
err_out_irq:
        free_irq(pdev->irq, mvi);
err_out_msi:
        if (mvi->flags |= MVF_MSI)
                pci_disable_msi(pdev);
err_out_mvi:
        mvs_free(mvi);
err_out_regions:
        pci_release_regions(pdev);
err_out_disable:
        pci_disable_device(pdev);
        return rc;
}

static void __devexit mvs_pci_remove(struct pci_dev *pdev)
{
        struct mvs_info *mvi = pci_get_drvdata(pdev);

        pci_set_drvdata(pdev, NULL);

        sas_unregister_ha(&mvi->sas);
        sas_remove_host(mvi->shost);
        scsi_remove_host(mvi->shost);

        free_irq(pdev->irq, mvi);
        if (mvi->flags & MVF_MSI)
                pci_disable_msi(pdev);
        mvs_free(mvi);
        pci_release_regions(pdev);
        pci_disable_device(pdev);
}

static struct sas_domain_function_template mvs_transport_ops = {
        .lldd_execute_task      = mvs_task_exec,
        .lldd_control_phy       = mvs_phy_control,
};

static struct pci_device_id __devinitdata mvs_pci_table[] = {
        { PCI_VDEVICE(MARVELL, 0x6320), chip_6320 },
        { PCI_VDEVICE(MARVELL, 0x6340), chip_6440 },
        { PCI_VDEVICE(MARVELL, 0x6440), chip_6440 },
        { PCI_VDEVICE(MARVELL, 0x6480), chip_6480 },

        { }     /* terminate list */
};

static struct pci_driver mvs_pci_driver = {
        .name           = DRV_NAME,
        .id_table       = mvs_pci_table,
        .probe          = mvs_pci_init,
        .remove         = __devexit_p(mvs_pci_remove),
};

static int __init mvs_init(void)
{
        int rc;

        mvs_stt = sas_domain_attach_transport(&mvs_transport_ops);
        if (!mvs_stt)
                return -ENOMEM;

        rc = pci_register_driver(&mvs_pci_driver);
        if (rc)
                goto err_out;

        return 0;

err_out:
        sas_release_transport(mvs_stt);
        return rc;
}

static void __exit mvs_exit(void)
{
        pci_unregister_driver(&mvs_pci_driver);
        sas_release_transport(mvs_stt);
}

module_init(mvs_init);
module_exit(mvs_exit);

MODULE_AUTHOR("Jeff Garzik <jgarzik@pobox.com>");
MODULE_DESCRIPTION("Marvell 88SE6440 SAS/SATA controller driver");
MODULE_VERSION(DRV_VERSION);
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, mvs_pci_table);