SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2009 ServerEngines
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@serverengines.com
  14  *
  15  *  ServerEngines
  16  * 209 N. Fair Oaks Ave
  17  * Sunnyvale, CA 94085
  18  *
  19  */
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/blkdev.h>
  24 #include <linux/pci.h>
  25 #include <linux/string.h>
  26 #include <linux/kernel.h>
  27 #include <linux/semaphore.h>
  28
  29 #include <scsi/libiscsi.h>
  30 #include <scsi/scsi_transport_iscsi.h>
  31 #include <scsi/scsi_transport.h>
  32 #include <scsi/scsi_cmnd.h>
  33 #include <scsi/scsi_device.h>
  34 #include <scsi/scsi_host.h>
  35 #include <scsi/scsi.h>
  36 #include "be_main.h"
  37 #include "be_iscsi.h"
  38 #include "be_mgmt.h"
  39
  40 static unsigned int be_iopoll_budget = 10;
  41 static unsigned int be_max_phys_size = 64;
  42 static unsigned int enable_msix = 1;
  43 static unsigned int ring_mode;
  44
  45 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  46 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  47 MODULE_AUTHOR("ServerEngines Corporation");
  48 MODULE_LICENSE("GPL");
  49 module_param(be_iopoll_budget, int, 0);
  50 module_param(enable_msix, int, 0);
  51 module_param(be_max_phys_size, uint, S_IRUGO);
  52 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
  53                                    "contiguous memory that can be allocated."
  54                                    "Range is 16 - 128");
  55
  56 static int beiscsi_slave_configure(struct scsi_device *sdev)
  57 {
  58         blk_queue_max_segment_size(sdev->request_queue, 65536);
  59         return 0;
  60 }
  61
  62 /*------------------- PCI Driver operations and data ----------------- */
  63 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
  64         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  65         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  66         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  67         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
  68         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID4) },
  69         { 0 }
  70 };
  71 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  72
  73 static struct scsi_host_template beiscsi_sht = {
  74         .module = THIS_MODULE,
  75         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
  76         .proc_name = DRV_NAME,
  77         .queuecommand = iscsi_queuecommand,
  78         .eh_abort_handler = iscsi_eh_abort,
  79         .change_queue_depth = iscsi_change_queue_depth,
  80         .slave_configure = beiscsi_slave_configure,
  81         .target_alloc = iscsi_target_alloc,
  82         .eh_device_reset_handler = iscsi_eh_device_reset,
  83         .eh_target_reset_handler = iscsi_eh_target_reset,
  84         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
  85         .can_queue = BE2_IO_DEPTH,
  86         .this_id = -1,
  87         .max_sectors = BEISCSI_MAX_SECTORS,
  88         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
  89         .use_clustering = ENABLE_CLUSTERING,
  90 };
  91
  92 static struct scsi_transport_template *beiscsi_scsi_transport;
  93
  94 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
  95 {
  96         struct beiscsi_hba *phba;
  97         struct Scsi_Host *shost;
  98
  99         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 100         if (!shost) {
 101                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
 102                         "iscsi_host_alloc failed \n");
 103                 return NULL;
 104         }
 105         shost->dma_boundary = pcidev->dma_mask;
 106         shost->max_id = BE2_MAX_SESSIONS;
 107         shost->max_channel = 0;
 108         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 109         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 110         shost->transportt = beiscsi_scsi_transport;
 111         phba = iscsi_host_priv(shost);
 112         memset(phba, 0, sizeof(*phba));
 113         phba->shost = shost;
 114         phba->pcidev = pci_dev_get(pcidev);
 115         pci_set_drvdata(pcidev, phba);
 116
 117         if (iscsi_host_add(shost, &phba->pcidev->dev))
 118                 goto free_devices;
 119         return phba;
 120
 121 free_devices:
 122         pci_dev_put(phba->pcidev);
 123         iscsi_host_free(phba->shost);
 124         return NULL;
 125 }
 126
 127 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 128 {
 129         if (phba->csr_va) {
 130                 iounmap(phba->csr_va);
 131                 phba->csr_va = NULL;
 132         }
 133         if (phba->db_va) {
 134                 iounmap(phba->db_va);
 135                 phba->db_va = NULL;
 136         }
 137         if (phba->pci_va) {
 138                 iounmap(phba->pci_va);
 139                 phba->pci_va = NULL;
 140         }
 141 }
 142
 143 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 144                                 struct pci_dev *pcidev)
 145 {
 146         u8 __iomem *addr;
 147
 148         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 149                                pci_resource_len(pcidev, 2));
 150         if (addr == NULL)
 151                 return -ENOMEM;
 152         phba->ctrl.csr = addr;
 153         phba->csr_va = addr;
 154         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 155
 156         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 157         if (addr == NULL)
 158                 goto pci_map_err;
 159         phba->ctrl.db = addr;
 160         phba->db_va = addr;
 161         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 162
 163         addr = ioremap_nocache(pci_resource_start(pcidev, 1),
 164                                pci_resource_len(pcidev, 1));
 165         if (addr == NULL)
 166                 goto pci_map_err;
 167         phba->ctrl.pcicfg = addr;
 168         phba->pci_va = addr;
 169         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, 1);
 170         return 0;
 171
 172 pci_map_err:
 173         beiscsi_unmap_pci_function(phba);
 174         return -ENOMEM;
 175 }
 176
 177 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 178 {
 179         int ret;
 180
 181         ret = pci_enable_device(pcidev);
 182         if (ret) {
 183                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
 184                         "failed. Returning -ENODEV\n");
 185                 return ret;
 186         }
 187
 188         pci_set_master(pcidev);
 189         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 190                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 191                 if (ret) {
 192                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 193                         pci_disable_device(pcidev);
 194                         return ret;
 195                 }
 196         }
 197         return 0;
 198 }
 199
 200 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 201 {
 202         struct be_ctrl_info *ctrl = &phba->ctrl;
 203         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 204         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 205         int status = 0;
 206
 207         ctrl->pdev = pdev;
 208         status = beiscsi_map_pci_bars(phba, pdev);
 209         if (status)
 210                 return status;
 211         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 212         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 213                                                   mbox_mem_alloc->size,
 214                                                   &mbox_mem_alloc->dma);
 215         if (!mbox_mem_alloc->va) {
 216                 beiscsi_unmap_pci_function(phba);
 217                 status = -ENOMEM;
 218                 return status;
 219         }
 220
 221         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 222         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 223         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 224         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 225         spin_lock_init(&ctrl->mbox_lock);
 226         spin_lock_init(&phba->ctrl.mcc_lock);
 227         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 228
 229         return status;
 230 }
 231
 232 static void beiscsi_get_params(struct beiscsi_hba *phba)
 233 {
 234         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 235                                     - (phba->fw_config.iscsi_cid_count
 236                                     + BE2_TMFS
 237                                     + BE2_NOPOUT_REQ));
 238         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 239         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count;;
 240         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;;
 241         phba->params.num_sge_per_io = BE2_SGE;
 242         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 243         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 244         phba->params.eq_timer = 64;
 245         phba->params.num_eq_entries =
 246             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 247                                     + BE2_TMFS) / 512) + 1) * 512;
 248         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 249                                 ? 1024 : phba->params.num_eq_entries;
 250         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
 251                              phba->params.num_eq_entries);
 252         phba->params.num_cq_entries =
 253             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 254                                     + BE2_TMFS) / 512) + 1) * 512;
 255         phba->params.wrbs_per_cxn = 256;
 256 }
 257
 258 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 259                            unsigned int id, unsigned int clr_interrupt,
 260                            unsigned int num_processed,
 261                            unsigned char rearm, unsigned char event)
 262 {
 263         u32 val = 0;
 264         val |= id & DB_EQ_RING_ID_MASK;
 265         if (rearm)
 266                 val |= 1 << DB_EQ_REARM_SHIFT;
 267         if (clr_interrupt)
 268                 val |= 1 << DB_EQ_CLR_SHIFT;
 269         if (event)
 270                 val |= 1 << DB_EQ_EVNT_SHIFT;
 271         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 272         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 273 }
 274
 275 /**
 276  * be_isr_mcc - The isr routine of the driver.
 277  * @irq: Not used
 278  * @dev_id: Pointer to host adapter structure
 279  */
 280 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 281 {
 282         struct beiscsi_hba *phba;
 283         struct be_eq_entry *eqe = NULL;
 284         struct be_queue_info *eq;
 285         struct be_queue_info *mcc;
 286         unsigned int num_eq_processed;
 287         struct be_eq_obj *pbe_eq;
 288         unsigned long flags;
 289
 290         pbe_eq = dev_id;
 291         eq = &pbe_eq->q;
 292         phba =  pbe_eq->phba;
 293         mcc = &phba->ctrl.mcc_obj.cq;
 294         eqe = queue_tail_node(eq);
 295         if (!eqe)
 296                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 297
 298         num_eq_processed = 0;
 299
 300         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 301                                 & EQE_VALID_MASK) {
 302                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 303                      resource_id) / 32] &
 304                      EQE_RESID_MASK) >> 16) == mcc->id) {
 305                         spin_lock_irqsave(&phba->isr_lock, flags);
 306                         phba->todo_mcc_cq = 1;
 307                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 308                 }
 309                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 310                 queue_tail_inc(eq);
 311                 eqe = queue_tail_node(eq);
 312                 num_eq_processed++;
 313         }
 314         if (phba->todo_mcc_cq)
 315                 queue_work(phba->wq, &phba->work_cqs);
 316         if (num_eq_processed)
 317                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 318
 319         return IRQ_HANDLED;
 320 }
 321
 322 /**
 323  * be_isr_msix - The isr routine of the driver.
 324  * @irq: Not used
 325  * @dev_id: Pointer to host adapter structure
 326  */
 327 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 328 {
 329         struct beiscsi_hba *phba;
 330         struct be_eq_entry *eqe = NULL;
 331         struct be_queue_info *eq;
 332         struct be_queue_info *cq;
 333         unsigned int num_eq_processed;
 334         struct be_eq_obj *pbe_eq;
 335         unsigned long flags;
 336
 337         pbe_eq = dev_id;
 338         eq = &pbe_eq->q;
 339         cq = pbe_eq->cq;
 340         eqe = queue_tail_node(eq);
 341         if (!eqe)
 342                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 343
 344         phba = pbe_eq->phba;
 345         num_eq_processed = 0;
 346         if (blk_iopoll_enabled) {
 347                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 348                                         & EQE_VALID_MASK) {
 349                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 350                                 blk_iopoll_sched(&pbe_eq->iopoll);
 351
 352                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 353                         queue_tail_inc(eq);
 354                         eqe = queue_tail_node(eq);
 355                         num_eq_processed++;
 356                 }
 357                 if (num_eq_processed)
 358                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 359
 360                 return IRQ_HANDLED;
 361         } else {
 362                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 363                                                 & EQE_VALID_MASK) {
 364                         spin_lock_irqsave(&phba->isr_lock, flags);
 365                         phba->todo_cq = 1;
 366                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 367                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 368                         queue_tail_inc(eq);
 369                         eqe = queue_tail_node(eq);
 370                         num_eq_processed++;
 371                 }
 372                 if (phba->todo_cq)
 373                         queue_work(phba->wq, &phba->work_cqs);
 374
 375                 if (num_eq_processed)
 376                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 377
 378                 return IRQ_HANDLED;
 379         }
 380 }
 381
 382 /**
 383  * be_isr - The isr routine of the driver.
 384  * @irq: Not used
 385  * @dev_id: Pointer to host adapter structure
 386  */
 387 static irqreturn_t be_isr(int irq, void *dev_id)
 388 {
 389         struct beiscsi_hba *phba;
 390         struct hwi_controller *phwi_ctrlr;
 391         struct hwi_context_memory *phwi_context;
 392         struct be_eq_entry *eqe = NULL;
 393         struct be_queue_info *eq;
 394         struct be_queue_info *cq;
 395         struct be_queue_info *mcc;
 396         unsigned long flags, index;
 397         unsigned int num_mcceq_processed, num_ioeq_processed;
 398         struct be_ctrl_info *ctrl;
 399         struct be_eq_obj *pbe_eq;
 400         int isr;
 401
 402         phba = dev_id;
 403         ctrl = &phba->ctrl;;
 404         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 405                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 406         if (!isr)
 407                 return IRQ_NONE;
 408
 409         phwi_ctrlr = phba->phwi_ctrlr;
 410         phwi_context = phwi_ctrlr->phwi_ctxt;
 411         pbe_eq = &phwi_context->be_eq[0];
 412
 413         eq = &phwi_context->be_eq[0].q;
 414         mcc = &phba->ctrl.mcc_obj.cq;
 415         index = 0;
 416         eqe = queue_tail_node(eq);
 417         if (!eqe)
 418                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 419
 420         num_ioeq_processed = 0;
 421         num_mcceq_processed = 0;
 422         if (blk_iopoll_enabled) {
 423                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 424                                         & EQE_VALID_MASK) {
 425                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 426                              resource_id) / 32] &
 427                              EQE_RESID_MASK) >> 16) == mcc->id) {
 428                                 spin_lock_irqsave(&phba->isr_lock, flags);
 429                                 phba->todo_mcc_cq = 1;
 430                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 431                                 num_mcceq_processed++;
 432                         } else {
 433                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 434                                         blk_iopoll_sched(&pbe_eq->iopoll);
 435                                 num_ioeq_processed++;
 436                         }
 437                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 438                         queue_tail_inc(eq);
 439                         eqe = queue_tail_node(eq);
 440                 }
 441                 if (num_ioeq_processed || num_mcceq_processed) {
 442                         if (phba->todo_mcc_cq)
 443                                 queue_work(phba->wq, &phba->work_cqs);
 444
 445                 if ((num_mcceq_processed) && (!num_ioeq_processed))
 446                                 hwi_ring_eq_db(phba, eq->id, 0,
 447                                               (num_ioeq_processed +
 448                                                num_mcceq_processed) , 1, 1);
 449                         else
 450                                 hwi_ring_eq_db(phba, eq->id, 0,
 451                                                (num_ioeq_processed +
 452                                                 num_mcceq_processed), 0, 1);
 453
 454                         return IRQ_HANDLED;
 455                 } else
 456                         return IRQ_NONE;
 457         } else {
 458                 cq = &phwi_context->be_cq[0];
 459                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 460                                                 & EQE_VALID_MASK) {
 461
 462                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 463                              resource_id) / 32] &
 464                              EQE_RESID_MASK) >> 16) != cq->id) {
 465                                 spin_lock_irqsave(&phba->isr_lock, flags);
 466                                 phba->todo_mcc_cq = 1;
 467                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 468                         } else {
 469                                 spin_lock_irqsave(&phba->isr_lock, flags);
 470                                 phba->todo_cq = 1;
 471                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 472                         }
 473                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 474                         queue_tail_inc(eq);
 475                         eqe = queue_tail_node(eq);
 476                         num_ioeq_processed++;
 477                 }
 478                 if (phba->todo_cq || phba->todo_mcc_cq)
 479                         queue_work(phba->wq, &phba->work_cqs);
 480
 481                 if (num_ioeq_processed) {
 482                         hwi_ring_eq_db(phba, eq->id, 0,
 483                                        num_ioeq_processed, 1, 1);
 484                         return IRQ_HANDLED;
 485                 } else
 486                         return IRQ_NONE;
 487         }
 488 }
 489
 490 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 491 {
 492         struct pci_dev *pcidev = phba->pcidev;
 493         struct hwi_controller *phwi_ctrlr;
 494         struct hwi_context_memory *phwi_context;
 495         int ret, msix_vec, i = 0;
 496         char desc[32];
 497
 498         phwi_ctrlr = phba->phwi_ctrlr;
 499         phwi_context = phwi_ctrlr->phwi_ctxt;
 500
 501         if (phba->msix_enabled) {
 502                 for (i = 0; i < phba->num_cpus; i++) {
 503                         sprintf(desc, "beiscsi_msix_%04x", i);
 504                         msix_vec = phba->msix_entries[i].vector;
 505                         ret = request_irq(msix_vec, be_isr_msix, 0, desc,
 506                                           &phwi_context->be_eq[i]);
 507                 }
 508                 msix_vec = phba->msix_entries[i].vector;
 509                 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
 510                                   &phwi_context->be_eq[i]);
 511         } else {
 512                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
 513                                   "beiscsi", phba);
 514                 if (ret) {
 515                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
 516                                      "Failed to register irq\\n");
 517                         return ret;
 518                 }
 519         }
 520         return 0;
 521 }
 522
 523 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
 524                            unsigned int id, unsigned int num_processed,
 525                            unsigned char rearm, unsigned char event)
 526 {
 527         u32 val = 0;
 528         val |= id & DB_CQ_RING_ID_MASK;
 529         if (rearm)
 530                 val |= 1 << DB_CQ_REARM_SHIFT;
 531         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
 532         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 533 }
 534
 535 static unsigned int
 536 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
 537                           struct beiscsi_hba *phba,
 538                           unsigned short cid,
 539                           struct pdu_base *ppdu,
 540                           unsigned long pdu_len,
 541                           void *pbuffer, unsigned long buf_len)
 542 {
 543         struct iscsi_conn *conn = beiscsi_conn->conn;
 544         struct iscsi_session *session = conn->session;
 545         struct iscsi_task *task;
 546         struct beiscsi_io_task *io_task;
 547         struct iscsi_hdr *login_hdr;
 548
 549         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
 550                                                 PDUBASE_OPCODE_MASK) {
 551         case ISCSI_OP_NOOP_IN:
 552                 pbuffer = NULL;
 553                 buf_len = 0;
 554                 break;
 555         case ISCSI_OP_ASYNC_EVENT:
 556                 break;
 557         case ISCSI_OP_REJECT:
 558                 WARN_ON(!pbuffer);
 559                 WARN_ON(!(buf_len == 48));
 560                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
 561                 break;
 562         case ISCSI_OP_LOGIN_RSP:
 563         case ISCSI_OP_TEXT_RSP:
 564                 task = conn->login_task;
 565                 io_task = task->dd_data;
 566                 login_hdr = (struct iscsi_hdr *)ppdu;
 567                 login_hdr->itt = io_task->libiscsi_itt;
 568                 break;
 569         default:
 570                 shost_printk(KERN_WARNING, phba->shost,
 571                              "Unrecognized opcode 0x%x in async msg \n",
 572                              (ppdu->
 573                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
 574                                                 & PDUBASE_OPCODE_MASK));
 575                 return 1;
 576         }
 577
 578         spin_lock_bh(&session->lock);
 579         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
 580         spin_unlock_bh(&session->lock);
 581         return 0;
 582 }
 583
 584 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 585 {
 586         struct sgl_handle *psgl_handle;
 587
 588         if (phba->io_sgl_hndl_avbl) {
 589                 SE_DEBUG(DBG_LVL_8,
 590                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
 591                          phba->io_sgl_alloc_index);
 592                 psgl_handle = phba->io_sgl_hndl_base[phba->
 593                                                 io_sgl_alloc_index];
 594                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
 595                 phba->io_sgl_hndl_avbl--;
 596                 if (phba->io_sgl_alloc_index == (phba->params.
 597                                                  ios_per_ctrl - 1))
 598                         phba->io_sgl_alloc_index = 0;
 599                 else
 600                         phba->io_sgl_alloc_index++;
 601         } else
 602                 psgl_handle = NULL;
 603         return psgl_handle;
 604 }
 605
 606 static void
 607 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 608 {
 609         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
 610                  phba->io_sgl_free_index);
 611         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 612                 /*
 613                  * this can happen if clean_task is called on a task that
 614                  * failed in xmit_task or alloc_pdu.
 615                  */
 616                  SE_DEBUG(DBG_LVL_8,
 617                          "Double Free in IO SGL io_sgl_free_index=%d,"
 618                          "value there=%p \n", phba->io_sgl_free_index,
 619                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
 620                 return;
 621         }
 622         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
 623         phba->io_sgl_hndl_avbl++;
 624         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
 625                 phba->io_sgl_free_index = 0;
 626         else
 627                 phba->io_sgl_free_index++;
 628 }
 629
 630 /**
 631  * alloc_wrb_handle - To allocate a wrb handle
 632  * @phba: The hba pointer
 633  * @cid: The cid to use for allocation
 634  *
 635  * This happens under session_lock until submission to chip
 636  */
 637 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
 638 {
 639         struct hwi_wrb_context *pwrb_context;
 640         struct hwi_controller *phwi_ctrlr;
 641         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
 642
 643         phwi_ctrlr = phba->phwi_ctrlr;
 644         pwrb_context = &phwi_ctrlr->wrb_context[cid];
 645         if (pwrb_context->wrb_handles_available >= 2) {
 646                 pwrb_handle = pwrb_context->pwrb_handle_base[
 647                                             pwrb_context->alloc_index];
 648                 pwrb_context->wrb_handles_available--;
 649                 if (pwrb_context->alloc_index ==
 650                                                 (phba->params.wrbs_per_cxn - 1))
 651                         pwrb_context->alloc_index = 0;
 652                 else
 653                         pwrb_context->alloc_index++;
 654
 655                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
 656                                                 pwrb_context->alloc_index];
 657                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
 658         } else
 659                 pwrb_handle = NULL;
 660         return pwrb_handle;
 661 }
 662
 663 /**
 664  * free_wrb_handle - To free the wrb handle back to pool
 665  * @phba: The hba pointer
 666  * @pwrb_context: The context to free from
 667  * @pwrb_handle: The wrb_handle to free
 668  *
 669  * This happens under session_lock until submission to chip
 670  */
 671 static void
 672 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
 673                 struct wrb_handle *pwrb_handle)
 674 {
 675         if (!ring_mode)
 676                 pwrb_context->pwrb_handle_base[pwrb_context->free_index] =
 677                                                pwrb_handle;
 678         pwrb_context->wrb_handles_available++;
 679         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
 680                 pwrb_context->free_index = 0;
 681         else
 682                 pwrb_context->free_index++;
 683
 684         SE_DEBUG(DBG_LVL_8,
 685                  "FREE WRB: pwrb_handle=%p free_index=0x%x"
 686                  "wrb_handles_available=%d \n",
 687                  pwrb_handle, pwrb_context->free_index,
 688                  pwrb_context->wrb_handles_available);
 689 }
 690
 691 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
 692 {
 693         struct sgl_handle *psgl_handle;
 694
 695         if (phba->eh_sgl_hndl_avbl) {
 696                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
 697                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
 698                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
 699                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
 700                 phba->eh_sgl_hndl_avbl--;
 701                 if (phba->eh_sgl_alloc_index ==
 702                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
 703                      1))
 704                         phba->eh_sgl_alloc_index = 0;
 705                 else
 706                         phba->eh_sgl_alloc_index++;
 707         } else
 708                 psgl_handle = NULL;
 709         return psgl_handle;
 710 }
 711
 712 void
 713 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 714 {
 715
 716         SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d \n",
 717                              phba->eh_sgl_free_index);
 718         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
 719                 /*
 720                  * this can happen if clean_task is called on a task that
 721                  * failed in xmit_task or alloc_pdu.
 722                  */
 723                 SE_DEBUG(DBG_LVL_8,
 724                          "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
 725                          phba->eh_sgl_free_index);
 726                 return;
 727         }
 728         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
 729         phba->eh_sgl_hndl_avbl++;
 730         if (phba->eh_sgl_free_index ==
 731             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
 732                 phba->eh_sgl_free_index = 0;
 733         else
 734                 phba->eh_sgl_free_index++;
 735 }
 736
 737 static void
 738 be_complete_io(struct beiscsi_conn *beiscsi_conn,
 739                struct iscsi_task *task, struct sol_cqe *psol)
 740 {
 741         struct beiscsi_io_task *io_task = task->dd_data;
 742         struct be_status_bhs *sts_bhs =
 743                                 (struct be_status_bhs *)io_task->cmd_bhs;
 744         struct iscsi_conn *conn = beiscsi_conn->conn;
 745         unsigned int sense_len;
 746         unsigned char *sense;
 747         u32 resid = 0, exp_cmdsn, max_cmdsn;
 748         u8 rsp, status, flags;
 749
 750         exp_cmdsn = (psol->
 751                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 752                         & SOL_EXP_CMD_SN_MASK);
 753         max_cmdsn = ((psol->
 754                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 755                         & SOL_EXP_CMD_SN_MASK) +
 756                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 757                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 758         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
 759                                                 & SOL_RESP_MASK) >> 16);
 760         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
 761                                                 & SOL_STS_MASK) >> 8);
 762         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 763                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 764
 765         task->sc->result = (DID_OK << 16) | status;
 766         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
 767                 task->sc->result = DID_ERROR << 16;
 768                 goto unmap;
 769         }
 770
 771         /* bidi not initially supported */
 772         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
 773                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
 774                                 32] & SOL_RES_CNT_MASK);
 775
 776                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
 777                         task->sc->result = DID_ERROR << 16;
 778
 779                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 780                         scsi_set_resid(task->sc, resid);
 781                         if (!status && (scsi_bufflen(task->sc) - resid <
 782                             task->sc->underflow))
 783                                 task->sc->result = DID_ERROR << 16;
 784                 }
 785         }
 786
 787         if (status == SAM_STAT_CHECK_CONDITION) {
 788                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
 789                 sense = sts_bhs->sense_info + sizeof(unsigned short);
 790                 sense_len =  cpu_to_be16(*slen);
 791                 memcpy(task->sc->sense_buffer, sense,
 792                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
 793         }
 794         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
 795                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
 796                                                         & SOL_RES_CNT_MASK)
 797                          conn->rxdata_octets += (psol->
 798                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
 799                              & SOL_RES_CNT_MASK);
 800         }
 801 unmap:
 802         scsi_dma_unmap(io_task->scsi_cmnd);
 803         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
 804 }
 805
 806 static void
 807 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
 808                    struct iscsi_task *task, struct sol_cqe *psol)
 809 {
 810         struct iscsi_logout_rsp *hdr;
 811         struct beiscsi_io_task *io_task = task->dd_data;
 812         struct iscsi_conn *conn = beiscsi_conn->conn;
 813
 814         hdr = (struct iscsi_logout_rsp *)task->hdr;
 815         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
 816         hdr->t2wait = 5;
 817         hdr->t2retain = 0;
 818         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 819                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 820         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
 821                                         32] & SOL_RESP_MASK);
 822         hdr->exp_cmdsn = cpu_to_be32(psol->
 823                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 824                                         & SOL_EXP_CMD_SN_MASK);
 825         hdr->max_cmdsn = be32_to_cpu((psol->
 826                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 827                                         & SOL_EXP_CMD_SN_MASK) +
 828                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 829                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 830         hdr->dlength[0] = 0;
 831         hdr->dlength[1] = 0;
 832         hdr->dlength[2] = 0;
 833         hdr->hlength = 0;
 834         hdr->itt = io_task->libiscsi_itt;
 835         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 836 }
 837
 838 static void
 839 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
 840                 struct iscsi_task *task, struct sol_cqe *psol)
 841 {
 842         struct iscsi_tm_rsp *hdr;
 843         struct iscsi_conn *conn = beiscsi_conn->conn;
 844         struct beiscsi_io_task *io_task = task->dd_data;
 845
 846         hdr = (struct iscsi_tm_rsp *)task->hdr;
 847         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
 848         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 849                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 850         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
 851                                         32] & SOL_RESP_MASK);
 852         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
 853                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
 854         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
 855                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
 856                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 857                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 858         hdr->itt = io_task->libiscsi_itt;
 859         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 860 }
 861
 862 static void
 863 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
 864                        struct beiscsi_hba *phba, struct sol_cqe *psol)
 865 {
 866         struct hwi_wrb_context *pwrb_context;
 867         struct wrb_handle *pwrb_handle = NULL;
 868         struct sgl_handle *psgl_handle = NULL;
 869         struct hwi_controller *phwi_ctrlr;
 870         struct iscsi_task *task;
 871         struct beiscsi_io_task *io_task;
 872         struct iscsi_conn *conn = beiscsi_conn->conn;
 873         struct iscsi_session *session = conn->session;
 874
 875         phwi_ctrlr = phba->phwi_ctrlr;
 876         if (ring_mode) {
 877                 psgl_handle = phba->sgl_hndl_array[((psol->
 878                               dw[offsetof(struct amap_sol_cqe_ring, icd_index) /
 879                                 32] & SOL_ICD_INDEX_MASK) >> 6)];
 880                 pwrb_context = &phwi_ctrlr->wrb_context[psgl_handle->cid];
 881                 task = psgl_handle->task;
 882                 pwrb_handle = NULL;
 883         } else {
 884                 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
 885                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
 886                                 SOL_CID_MASK) >> 6) -
 887                                 phba->fw_config.iscsi_cid_start];
 888                 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
 889                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
 890                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
 891                 task = pwrb_handle->pio_handle;
 892         }
 893
 894         io_task = task->dd_data;
 895         spin_lock(&phba->mgmt_sgl_lock);
 896         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
 897         spin_unlock(&phba->mgmt_sgl_lock);
 898         spin_lock_bh(&session->lock);
 899         free_wrb_handle(phba, pwrb_context, pwrb_handle);
 900         spin_unlock_bh(&session->lock);
 901 }
 902
 903 static void
 904 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
 905                        struct iscsi_task *task, struct sol_cqe *psol)
 906 {
 907         struct iscsi_nopin *hdr;
 908         struct iscsi_conn *conn = beiscsi_conn->conn;
 909         struct beiscsi_io_task *io_task = task->dd_data;
 910
 911         hdr = (struct iscsi_nopin *)task->hdr;
 912         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 913                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 914         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
 915                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
 916         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
 917                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
 918                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 919                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 920         hdr->opcode = ISCSI_OP_NOOP_IN;
 921         hdr->itt = io_task->libiscsi_itt;
 922         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 923 }
 924
 925 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
 926                              struct beiscsi_hba *phba, struct sol_cqe *psol)
 927 {
 928         struct hwi_wrb_context *pwrb_context;
 929         struct wrb_handle *pwrb_handle;
 930         struct iscsi_wrb *pwrb = NULL;
 931         struct hwi_controller *phwi_ctrlr;
 932         struct iscsi_task *task;
 933         struct sgl_handle *psgl_handle = NULL;
 934         unsigned int type;
 935         struct iscsi_conn *conn = beiscsi_conn->conn;
 936         struct iscsi_session *session = conn->session;
 937
 938         phwi_ctrlr = phba->phwi_ctrlr;
 939         if (ring_mode) {
 940                 psgl_handle = phba->sgl_hndl_array[((psol->
 941                               dw[offsetof(struct amap_sol_cqe_ring, icd_index) /
 942                               32] & SOL_ICD_INDEX_MASK) >> 6)];
 943                 task = psgl_handle->task;
 944                 type = psgl_handle->type;
 945         } else {
 946                 pwrb_context = &phwi_ctrlr->
 947                                 wrb_context[((psol->dw[offsetof
 948                                 (struct amap_sol_cqe, cid) / 32]
 949                                 & SOL_CID_MASK) >> 6) -
 950                                 phba->fw_config.iscsi_cid_start];
 951                 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
 952                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
 953                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
 954                 task = pwrb_handle->pio_handle;
 955                 pwrb = pwrb_handle->pwrb;
 956                 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
 957                          WRB_TYPE_MASK) >> 28;
 958         }
 959         spin_lock_bh(&session->lock);
 960         switch (type) {
 961         case HWH_TYPE_IO:
 962         case HWH_TYPE_IO_RD:
 963                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
 964                     ISCSI_OP_NOOP_OUT) {
 965                         be_complete_nopin_resp(beiscsi_conn, task, psol);
 966                 } else
 967                         be_complete_io(beiscsi_conn, task, psol);
 968                 break;
 969
 970         case HWH_TYPE_LOGOUT:
 971                 be_complete_logout(beiscsi_conn, task, psol);
 972                 break;
 973
 974         case HWH_TYPE_LOGIN:
 975                 SE_DEBUG(DBG_LVL_1,
 976                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
 977                          "- Solicited path \n");
 978                 break;
 979
 980         case HWH_TYPE_TMF:
 981                 be_complete_tmf(beiscsi_conn, task, psol);
 982                 break;
 983
 984         case HWH_TYPE_NOP:
 985                 be_complete_nopin_resp(beiscsi_conn, task, psol);
 986                 break;
 987
 988         default:
 989                 if (ring_mode)
 990                         shost_printk(KERN_WARNING, phba->shost,
 991                                 "In hwi_complete_cmd, unknown type = %d"
 992                                 "icd_index 0x%x CID 0x%x\n", type,
 993                                 ((psol->dw[offsetof(struct amap_sol_cqe_ring,
 994                                 icd_index) / 32] & SOL_ICD_INDEX_MASK) >> 6),
 995                                 psgl_handle->cid);
 996                 else
 997                         shost_printk(KERN_WARNING, phba->shost,
 998                                 "In hwi_complete_cmd, unknown type = %d"
 999                                 "wrb_index 0x%x CID 0x%x\n", type,
1000                                 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1001                                 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1002                                 ((psol->dw[offsetof(struct amap_sol_cqe,
1003                                 cid) / 32] & SOL_CID_MASK) >> 6));
1004                 break;
1005         }
1006
1007         spin_unlock_bh(&session->lock);
1008 }
1009
1010 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1011                                           *pasync_ctx, unsigned int is_header,
1012                                           unsigned int host_write_ptr)
1013 {
1014         if (is_header)
1015                 return &pasync_ctx->async_entry[host_write_ptr].
1016                     header_busy_list;
1017         else
1018                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1019 }
1020
1021 static struct async_pdu_handle *
1022 hwi_get_async_handle(struct beiscsi_hba *phba,
1023                      struct beiscsi_conn *beiscsi_conn,
1024                      struct hwi_async_pdu_context *pasync_ctx,
1025                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1026 {
1027         struct be_bus_address phys_addr;
1028         struct list_head *pbusy_list;
1029         struct async_pdu_handle *pasync_handle = NULL;
1030         int buffer_len = 0;
1031         unsigned char buffer_index = -1;
1032         unsigned char is_header = 0;
1033
1034         phys_addr.u.a32.address_lo =
1035             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1036             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1037                                                 & PDUCQE_DPL_MASK) >> 16);
1038         phys_addr.u.a32.address_hi =
1039             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1040
1041         phys_addr.u.a64.address =
1042                         *((unsigned long long *)(&phys_addr.u.a64.address));
1043
1044         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1045                         & PDUCQE_CODE_MASK) {
1046         case UNSOL_HDR_NOTIFY:
1047                 is_header = 1;
1048
1049                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1050                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1051                         index) / 32] & PDUCQE_INDEX_MASK));
1052
1053                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1054                                 pasync_ctx->async_header.pa_base.u.a64.address);
1055
1056                 buffer_index = buffer_len /
1057                                 pasync_ctx->async_header.buffer_size;
1058
1059                 break;
1060         case UNSOL_DATA_NOTIFY:
1061                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1062                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1063                                         index) / 32] & PDUCQE_INDEX_MASK));
1064                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1065                                         pasync_ctx->async_data.pa_base.u.
1066                                         a64.address);
1067                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1068                 break;
1069         default:
1070                 pbusy_list = NULL;
1071                 shost_printk(KERN_WARNING, phba->shost,
1072                         "Unexpected code=%d \n",
1073                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1074                                         code) / 32] & PDUCQE_CODE_MASK);
1075                 return NULL;
1076         }
1077
1078         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1079         WARN_ON(list_empty(pbusy_list));
1080         list_for_each_entry(pasync_handle, pbusy_list, link) {
1081                 WARN_ON(pasync_handle->consumed);
1082                 if (pasync_handle->index == buffer_index)
1083                         break;
1084         }
1085
1086         WARN_ON(!pasync_handle);
1087
1088         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1089                                              phba->fw_config.iscsi_cid_start;
1090         pasync_handle->is_header = is_header;
1091         pasync_handle->buffer_len = ((pdpdu_cqe->
1092                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1093                         & PDUCQE_DPL_MASK) >> 16);
1094
1095         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1096                         index) / 32] & PDUCQE_INDEX_MASK);
1097         return pasync_handle;
1098 }
1099
1100 static unsigned int
1101 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1102                            unsigned int is_header, unsigned int cq_index)
1103 {
1104         struct list_head *pbusy_list;
1105         struct async_pdu_handle *pasync_handle;
1106         unsigned int num_entries, writables = 0;
1107         unsigned int *pep_read_ptr, *pwritables;
1108
1109
1110         if (is_header) {
1111                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1112                 pwritables = &pasync_ctx->async_header.writables;
1113                 num_entries = pasync_ctx->async_header.num_entries;
1114         } else {
1115                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1116                 pwritables = &pasync_ctx->async_data.writables;
1117                 num_entries = pasync_ctx->async_data.num_entries;
1118         }
1119
1120         while ((*pep_read_ptr) != cq_index) {
1121                 (*pep_read_ptr)++;
1122                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1123
1124                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1125                                                      *pep_read_ptr);
1126                 if (writables == 0)
1127                         WARN_ON(list_empty(pbusy_list));
1128
1129                 if (!list_empty(pbusy_list)) {
1130                         pasync_handle = list_entry(pbusy_list->next,
1131                                                    struct async_pdu_handle,
1132                                                    link);
1133                         WARN_ON(!pasync_handle);
1134                         pasync_handle->consumed = 1;
1135                 }
1136
1137                 writables++;
1138         }
1139
1140         if (!writables) {
1141                 SE_DEBUG(DBG_LVL_1,
1142                          "Duplicate notification received - index 0x%x!!\n",
1143                          cq_index);
1144                 WARN_ON(1);
1145         }
1146
1147         *pwritables = *pwritables + writables;
1148         return 0;
1149 }
1150
1151 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1152                                        unsigned int cri)
1153 {
1154         struct hwi_controller *phwi_ctrlr;
1155         struct hwi_async_pdu_context *pasync_ctx;
1156         struct async_pdu_handle *pasync_handle, *tmp_handle;
1157         struct list_head *plist;
1158         unsigned int i = 0;
1159
1160         phwi_ctrlr = phba->phwi_ctrlr;
1161         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1162
1163         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1164
1165         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1166                 list_del(&pasync_handle->link);
1167
1168                 if (i == 0) {
1169                         list_add_tail(&pasync_handle->link,
1170                                       &pasync_ctx->async_header.free_list);
1171                         pasync_ctx->async_header.free_entries++;
1172                         i++;
1173                 } else {
1174                         list_add_tail(&pasync_handle->link,
1175                                       &pasync_ctx->async_data.free_list);
1176                         pasync_ctx->async_data.free_entries++;
1177                         i++;
1178                 }
1179         }
1180
1181         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1182         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1183         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1184         return 0;
1185 }
1186
1187 static struct phys_addr *
1188 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1189                      unsigned int is_header, unsigned int host_write_ptr)
1190 {
1191         struct phys_addr *pasync_sge = NULL;
1192
1193         if (is_header)
1194                 pasync_sge = pasync_ctx->async_header.ring_base;
1195         else
1196                 pasync_sge = pasync_ctx->async_data.ring_base;
1197
1198         return pasync_sge + host_write_ptr;
1199 }
1200
1201 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1202                                    unsigned int is_header)
1203 {
1204         struct hwi_controller *phwi_ctrlr;
1205         struct hwi_async_pdu_context *pasync_ctx;
1206         struct async_pdu_handle *pasync_handle;
1207         struct list_head *pfree_link, *pbusy_list;
1208         struct phys_addr *pasync_sge;
1209         unsigned int ring_id, num_entries;
1210         unsigned int host_write_num;
1211         unsigned int writables;
1212         unsigned int i = 0;
1213         u32 doorbell = 0;
1214
1215         phwi_ctrlr = phba->phwi_ctrlr;
1216         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1217
1218         if (is_header) {
1219                 num_entries = pasync_ctx->async_header.num_entries;
1220                 writables = min(pasync_ctx->async_header.writables,
1221                                 pasync_ctx->async_header.free_entries);
1222                 pfree_link = pasync_ctx->async_header.free_list.next;
1223                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1224                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1225         } else {
1226                 num_entries = pasync_ctx->async_data.num_entries;
1227                 writables = min(pasync_ctx->async_data.writables,
1228                                 pasync_ctx->async_data.free_entries);
1229                 pfree_link = pasync_ctx->async_data.free_list.next;
1230                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1231                 ring_id = phwi_ctrlr->default_pdu_data.id;
1232         }
1233
1234         writables = (writables / 8) * 8;
1235         if (writables) {
1236                 for (i = 0; i < writables; i++) {
1237                         pbusy_list =
1238                             hwi_get_async_busy_list(pasync_ctx, is_header,
1239                                                     host_write_num);
1240                         pasync_handle =
1241                             list_entry(pfree_link, struct async_pdu_handle,
1242                                                                 link);
1243                         WARN_ON(!pasync_handle);
1244                         pasync_handle->consumed = 0;
1245
1246                         pfree_link = pfree_link->next;
1247
1248                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1249                                                 is_header, host_write_num);
1250
1251                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1252                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1253
1254                         list_move(&pasync_handle->link, pbusy_list);
1255
1256                         host_write_num++;
1257                         host_write_num = host_write_num % num_entries;
1258                 }
1259
1260                 if (is_header) {
1261                         pasync_ctx->async_header.host_write_ptr =
1262                                                         host_write_num;
1263                         pasync_ctx->async_header.free_entries -= writables;
1264                         pasync_ctx->async_header.writables -= writables;
1265                         pasync_ctx->async_header.busy_entries += writables;
1266                 } else {
1267                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1268                         pasync_ctx->async_data.free_entries -= writables;
1269                         pasync_ctx->async_data.writables -= writables;
1270                         pasync_ctx->async_data.busy_entries += writables;
1271                 }
1272
1273                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1274                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1275                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1276                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1277                                         << DB_DEF_PDU_CQPROC_SHIFT;
1278
1279                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1280         }
1281 }
1282
1283 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1284                                          struct beiscsi_conn *beiscsi_conn,
1285                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1286 {
1287         struct hwi_controller *phwi_ctrlr;
1288         struct hwi_async_pdu_context *pasync_ctx;
1289         struct async_pdu_handle *pasync_handle = NULL;
1290         unsigned int cq_index = -1;
1291
1292         phwi_ctrlr = phba->phwi_ctrlr;
1293         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1294
1295         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1296                                              pdpdu_cqe, &cq_index);
1297         BUG_ON(pasync_handle->is_header != 0);
1298         if (pasync_handle->consumed == 0)
1299                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1300                                            cq_index);
1301
1302         hwi_free_async_msg(phba, pasync_handle->cri);
1303         hwi_post_async_buffers(phba, pasync_handle->is_header);
1304 }
1305
1306 static unsigned int
1307 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1308                   struct beiscsi_hba *phba,
1309                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1310 {
1311         struct list_head *plist;
1312         struct async_pdu_handle *pasync_handle;
1313         void *phdr = NULL;
1314         unsigned int hdr_len = 0, buf_len = 0;
1315         unsigned int status, index = 0, offset = 0;
1316         void *pfirst_buffer = NULL;
1317         unsigned int num_buf = 0;
1318
1319         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1320
1321         list_for_each_entry(pasync_handle, plist, link) {
1322                 if (index == 0) {
1323                         phdr = pasync_handle->pbuffer;
1324                         hdr_len = pasync_handle->buffer_len;
1325                 } else {
1326                         buf_len = pasync_handle->buffer_len;
1327                         if (!num_buf) {
1328                                 pfirst_buffer = pasync_handle->pbuffer;
1329                                 num_buf++;
1330                         }
1331                         memcpy(pfirst_buffer + offset,
1332                                pasync_handle->pbuffer, buf_len);
1333                         offset = buf_len;
1334                 }
1335                 index++;
1336         }
1337
1338         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1339                                            (beiscsi_conn->beiscsi_conn_cid -
1340                                             phba->fw_config.iscsi_cid_start),
1341                                             phdr, hdr_len, pfirst_buffer,
1342                                             buf_len);
1343
1344         if (status == 0)
1345                 hwi_free_async_msg(phba, cri);
1346         return 0;
1347 }
1348
1349 static unsigned int
1350 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1351                      struct beiscsi_hba *phba,
1352                      struct async_pdu_handle *pasync_handle)
1353 {
1354         struct hwi_async_pdu_context *pasync_ctx;
1355         struct hwi_controller *phwi_ctrlr;
1356         unsigned int bytes_needed = 0, status = 0;
1357         unsigned short cri = pasync_handle->cri;
1358         struct pdu_base *ppdu;
1359
1360         phwi_ctrlr = phba->phwi_ctrlr;
1361         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1362
1363         list_del(&pasync_handle->link);
1364         if (pasync_handle->is_header) {
1365                 pasync_ctx->async_header.busy_entries--;
1366                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1367                         hwi_free_async_msg(phba, cri);
1368                         BUG();
1369                 }
1370
1371                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1372                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1373                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1374                                 (unsigned short)pasync_handle->buffer_len;
1375                 list_add_tail(&pasync_handle->link,
1376                               &pasync_ctx->async_entry[cri].wait_queue.list);
1377
1378                 ppdu = pasync_handle->pbuffer;
1379                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1380                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1381                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1382                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1383                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1384
1385                 if (status == 0) {
1386                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1387                             bytes_needed;
1388
1389                         if (bytes_needed == 0)
1390                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1391                                                            pasync_ctx, cri);
1392                 }
1393         } else {
1394                 pasync_ctx->async_data.busy_entries--;
1395                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1396                         list_add_tail(&pasync_handle->link,
1397                                       &pasync_ctx->async_entry[cri].wait_queue.
1398                                       list);
1399                         pasync_ctx->async_entry[cri].wait_queue.
1400                                 bytes_received +=
1401                                 (unsigned short)pasync_handle->buffer_len;
1402
1403                         if (pasync_ctx->async_entry[cri].wait_queue.
1404                             bytes_received >=
1405                             pasync_ctx->async_entry[cri].wait_queue.
1406                             bytes_needed)
1407                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1408                                                            pasync_ctx, cri);
1409                 }
1410         }
1411         return status;
1412 }
1413
1414 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1415                                          struct beiscsi_hba *phba,
1416                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1417 {
1418         struct hwi_controller *phwi_ctrlr;
1419         struct hwi_async_pdu_context *pasync_ctx;
1420         struct async_pdu_handle *pasync_handle = NULL;
1421         unsigned int cq_index = -1;
1422
1423         phwi_ctrlr = phba->phwi_ctrlr;
1424         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1425         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1426                                              pdpdu_cqe, &cq_index);
1427
1428         if (pasync_handle->consumed == 0)
1429                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1430                                            cq_index);
1431         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1432         hwi_post_async_buffers(phba, pasync_handle->is_header);
1433 }
1434
1435
1436 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1437 {
1438         struct be_queue_info *cq;
1439         struct sol_cqe *sol;
1440         struct dmsg_cqe *dmsg;
1441         unsigned int num_processed = 0;
1442         unsigned int tot_nump = 0;
1443         struct beiscsi_conn *beiscsi_conn;
1444         struct sgl_handle *psgl_handle = NULL;
1445         struct beiscsi_endpoint *beiscsi_ep;
1446         struct iscsi_endpoint *ep;
1447         struct beiscsi_hba *phba;
1448
1449         cq = pbe_eq->cq;
1450         sol = queue_tail_node(cq);
1451         phba = pbe_eq->phba;
1452
1453         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1454                CQE_VALID_MASK) {
1455                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1456
1457                 if (ring_mode) {
1458                         psgl_handle = phba->sgl_hndl_array[((sol->
1459                                       dw[offsetof(struct amap_sol_cqe_ring,
1460                                       icd_index) / 32] & SOL_ICD_INDEX_MASK)
1461                                       >> 6)];
1462                         ep = phba->ep_array[psgl_handle->cid];
1463                 } else {
1464                         ep = phba->ep_array[(u32) ((sol->
1465                                    dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1466                                    SOL_CID_MASK) >> 6) -
1467                                    phba->fw_config.iscsi_cid_start];
1468                 }
1469                 beiscsi_ep = ep->dd_data;
1470                 beiscsi_conn = beiscsi_ep->conn;
1471                 if (num_processed >= 32) {
1472                         hwi_ring_cq_db(phba, cq->id,
1473                                         num_processed, 0, 0);
1474                         tot_nump += num_processed;
1475                         num_processed = 0;
1476                 }
1477
1478                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1479                         32] & CQE_CODE_MASK) {
1480                 case SOL_CMD_COMPLETE:
1481                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1482                         break;
1483                 case DRIVERMSG_NOTIFY:
1484                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1485                         dmsg = (struct dmsg_cqe *)sol;
1486                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1487                         break;
1488                 case UNSOL_HDR_NOTIFY:
1489                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1490                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1491                                              (struct i_t_dpdu_cqe *)sol);
1492                         break;
1493                 case UNSOL_DATA_NOTIFY:
1494                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1495                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1496                                              (struct i_t_dpdu_cqe *)sol);
1497                         break;
1498                 case CXN_INVALIDATE_INDEX_NOTIFY:
1499                 case CMD_INVALIDATED_NOTIFY:
1500                 case CXN_INVALIDATE_NOTIFY:
1501                         SE_DEBUG(DBG_LVL_1,
1502                                  "Ignoring CQ Error notification for cmd/cxn"
1503                                  "invalidate\n");
1504                         break;
1505                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1506                 case CMD_KILLED_INVALID_STATSN_RCVD:
1507                 case CMD_KILLED_INVALID_R2T_RCVD:
1508                 case CMD_CXN_KILLED_LUN_INVALID:
1509                 case CMD_CXN_KILLED_ICD_INVALID:
1510                 case CMD_CXN_KILLED_ITT_INVALID:
1511                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1512                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1513                         if (ring_mode) {
1514                                 SE_DEBUG(DBG_LVL_1,
1515                                  "CQ Error notification for cmd.. "
1516                                  "code %d cid 0x%x\n",
1517                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1518                                  32] & CQE_CODE_MASK, psgl_handle->cid);
1519                         } else {
1520                                 SE_DEBUG(DBG_LVL_1,
1521                                  "CQ Error notification for cmd.. "
1522                                  "code %d cid 0x%x\n",
1523                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1524                                  32] & CQE_CODE_MASK,
1525                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1526                                  32] & SOL_CID_MASK));
1527                         }
1528                         break;
1529                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1530                         SE_DEBUG(DBG_LVL_1,
1531                                  "Digest error on def pdu ring, dropping..\n");
1532                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1533                                              (struct i_t_dpdu_cqe *) sol);
1534                         break;
1535                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1536                 case CXN_KILLED_BURST_LEN_MISMATCH:
1537                 case CXN_KILLED_AHS_RCVD:
1538                 case CXN_KILLED_HDR_DIGEST_ERR:
1539                 case CXN_KILLED_UNKNOWN_HDR:
1540                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1541                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1542                 case CXN_KILLED_TIMED_OUT:
1543                 case CXN_KILLED_FIN_RCVD:
1544                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1545                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1546                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1547                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1548                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1549                         if (ring_mode) {
1550                                 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1551                                  "0x%x...\n",
1552                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1553                                  32] & CQE_CODE_MASK, psgl_handle->cid);
1554                         } else {
1555                                 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1556                                  "0x%x...\n",
1557                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1558                                  32] & CQE_CODE_MASK,
1559                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1560                                  32] & CQE_CID_MASK));
1561                         }
1562                         iscsi_conn_failure(beiscsi_conn->conn,
1563                                            ISCSI_ERR_CONN_FAILED);
1564                         break;
1565                 case CXN_KILLED_RST_SENT:
1566                 case CXN_KILLED_RST_RCVD:
1567                         if (ring_mode) {
1568                                 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1569                                 "received/sent on CID 0x%x...\n",
1570                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1571                                  32] & CQE_CODE_MASK, psgl_handle->cid);
1572                         } else {
1573                                 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1574                                 "received/sent on CID 0x%x...\n",
1575                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1576                                  32] & CQE_CODE_MASK,
1577                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1578                                  32] & CQE_CID_MASK));
1579                         }
1580                         iscsi_conn_failure(beiscsi_conn->conn,
1581                                            ISCSI_ERR_CONN_FAILED);
1582                         break;
1583                 default:
1584                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1585                                  "received on CID 0x%x...\n",
1586                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1587                                  32] & CQE_CODE_MASK,
1588                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1589                                  32] & CQE_CID_MASK));
1590                         break;
1591                 }
1592
1593                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1594                 queue_tail_inc(cq);
1595                 sol = queue_tail_node(cq);
1596                 num_processed++;
1597         }
1598
1599         if (num_processed > 0) {
1600                 tot_nump += num_processed;
1601                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1602         }
1603         return tot_nump;
1604 }
1605
1606 static void beiscsi_process_all_cqs(struct work_struct *work)
1607 {
1608         unsigned long flags;
1609         struct hwi_controller *phwi_ctrlr;
1610         struct hwi_context_memory *phwi_context;
1611         struct be_eq_obj *pbe_eq;
1612         struct beiscsi_hba *phba =
1613             container_of(work, struct beiscsi_hba, work_cqs);
1614
1615         phwi_ctrlr = phba->phwi_ctrlr;
1616         phwi_context = phwi_ctrlr->phwi_ctxt;
1617         if (phba->msix_enabled)
1618                 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1619         else
1620                 pbe_eq = &phwi_context->be_eq[0];
1621
1622         if (phba->todo_mcc_cq) {
1623                 spin_lock_irqsave(&phba->isr_lock, flags);
1624                 phba->todo_mcc_cq = 0;
1625                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1626         }
1627
1628         if (phba->todo_cq) {
1629                 spin_lock_irqsave(&phba->isr_lock, flags);
1630                 phba->todo_cq = 0;
1631                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1632                 beiscsi_process_cq(pbe_eq);
1633         }
1634 }
1635
1636 static int be_iopoll(struct blk_iopoll *iop, int budget)
1637 {
1638         static unsigned int ret;
1639         struct beiscsi_hba *phba;
1640         struct be_eq_obj *pbe_eq;
1641
1642         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1643         ret = beiscsi_process_cq(pbe_eq);
1644         if (ret < budget) {
1645                 phba = pbe_eq->phba;
1646                 blk_iopoll_complete(iop);
1647                 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1648                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1649         }
1650         return ret;
1651 }
1652
1653 static void
1654 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1655               unsigned int num_sg, struct beiscsi_io_task *io_task)
1656 {
1657         struct iscsi_sge *psgl;
1658         unsigned short sg_len, index;
1659         unsigned int sge_len = 0;
1660         unsigned long long addr;
1661         struct scatterlist *l_sg;
1662         unsigned int offset;
1663
1664         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1665                                       io_task->bhs_pa.u.a32.address_lo);
1666         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1667                                       io_task->bhs_pa.u.a32.address_hi);
1668
1669         l_sg = sg;
1670         for (index = 0; (index < num_sg) && (index < 2); index++, sg_next(sg)) {
1671                 if (index == 0) {
1672                         sg_len = sg_dma_len(sg);
1673                         addr = (u64) sg_dma_address(sg);
1674                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1675                                                         (addr & 0xFFFFFFFF));
1676                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1677                                                         (addr >> 32));
1678                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1679                                                         sg_len);
1680                         sge_len = sg_len;
1681                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1682                                                         1);
1683                 } else {
1684                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1685                                                         0);
1686                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1687                                                         pwrb, sge_len);
1688                         sg_len = sg_dma_len(sg);
1689                         addr = (u64) sg_dma_address(sg);
1690                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1691                                                         (addr & 0xFFFFFFFF));
1692                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1693                                                         (addr >> 32));
1694                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1695                                                         sg_len);
1696                 }
1697         }
1698         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1699         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1700
1701         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1702
1703         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1704                         io_task->bhs_pa.u.a32.address_hi);
1705         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1706                         io_task->bhs_pa.u.a32.address_lo);
1707
1708         if (num_sg == 2)
1709                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1);
1710         sg = l_sg;
1711         psgl++;
1712         psgl++;
1713         offset = 0;
1714         for (index = 0; index < num_sg; index++, sg_next(sg), psgl++) {
1715                 sg_len = sg_dma_len(sg);
1716                 addr = (u64) sg_dma_address(sg);
1717                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1718                                                 (addr & 0xFFFFFFFF));
1719                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1720                                                 (addr >> 32));
1721                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1722                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1723                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1724                 offset += sg_len;
1725         }
1726         psgl--;
1727         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1728 }
1729
1730 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1731 {
1732         struct iscsi_sge *psgl;
1733         unsigned long long addr;
1734         struct beiscsi_io_task *io_task = task->dd_data;
1735         struct beiscsi_conn *beiscsi_conn = io_task->conn;
1736         struct beiscsi_hba *phba = beiscsi_conn->phba;
1737
1738         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1739         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1740                                 io_task->bhs_pa.u.a32.address_lo);
1741         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1742                                 io_task->bhs_pa.u.a32.address_hi);
1743
1744         if (task->data) {
1745                 if (task->data_count) {
1746                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1747                         addr = (u64) pci_map_single(phba->pcidev,
1748                                                     task->data,
1749                                                     task->data_count, 1);
1750                 } else {
1751                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1752                         addr = 0;
1753                 }
1754                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1755                                                 (addr & 0xFFFFFFFF));
1756                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1757                                                 (addr >> 32));
1758                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1759                                                 task->data_count);
1760
1761                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1762         } else {
1763                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1764                 addr = 0;
1765         }
1766
1767         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1768
1769         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1770
1771         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1772                       io_task->bhs_pa.u.a32.address_hi);
1773         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1774                       io_task->bhs_pa.u.a32.address_lo);
1775         if (task->data) {
1776                 psgl++;
1777                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1778                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1779                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1780                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1781                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1782                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1783
1784                 psgl++;
1785                 if (task->data) {
1786                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1787                                                 (addr & 0xFFFFFFFF));
1788                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1789                                                 (addr >> 32));
1790                 }
1791                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1792         }
1793         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1794 }
1795
1796 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1797 {
1798         unsigned int num_cq_pages, num_async_pdu_buf_pages;
1799         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1800         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1801
1802         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1803                                       sizeof(struct sol_cqe));
1804         num_async_pdu_buf_pages =
1805                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1806                                        phba->params.defpdu_hdr_sz);
1807         num_async_pdu_buf_sgl_pages =
1808                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1809                                        sizeof(struct phys_addr));
1810         num_async_pdu_data_pages =
1811                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1812                                        phba->params.defpdu_data_sz);
1813         num_async_pdu_data_sgl_pages =
1814                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1815                                        sizeof(struct phys_addr));
1816
1817         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1818
1819         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1820                                                  BE_ISCSI_PDU_HEADER_SIZE;
1821         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1822                                             sizeof(struct hwi_context_memory);
1823
1824
1825         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1826             * (phba->params.wrbs_per_cxn)
1827             * phba->params.cxns_per_ctrl;
1828         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
1829                                  (phba->params.wrbs_per_cxn);
1830         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1831                                 phba->params.cxns_per_ctrl);
1832
1833         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1834                 phba->params.icds_per_ctrl;
1835         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1836                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1837
1838         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1839                 num_async_pdu_buf_pages * PAGE_SIZE;
1840         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1841                 num_async_pdu_data_pages * PAGE_SIZE;
1842         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1843                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1844         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1845                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1846         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1847                 phba->params.asyncpdus_per_ctrl *
1848                 sizeof(struct async_pdu_handle);
1849         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1850                 phba->params.asyncpdus_per_ctrl *
1851                 sizeof(struct async_pdu_handle);
1852         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1853                 sizeof(struct hwi_async_pdu_context) +
1854                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1855 }
1856
1857 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1858 {
1859         struct be_mem_descriptor *mem_descr;
1860         dma_addr_t bus_add;
1861         struct mem_array *mem_arr, *mem_arr_orig;
1862         unsigned int i, j, alloc_size, curr_alloc_size;
1863
1864         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1865         if (!phba->phwi_ctrlr)
1866                 return -ENOMEM;
1867
1868         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1869                                  GFP_KERNEL);
1870         if (!phba->init_mem) {
1871                 kfree(phba->phwi_ctrlr);
1872                 return -ENOMEM;
1873         }
1874
1875         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1876                                GFP_KERNEL);
1877         if (!mem_arr_orig) {
1878                 kfree(phba->init_mem);
1879                 kfree(phba->phwi_ctrlr);
1880                 return -ENOMEM;
1881         }
1882
1883         mem_descr = phba->init_mem;
1884         for (i = 0; i < SE_MEM_MAX; i++) {
1885                 j = 0;
1886                 mem_arr = mem_arr_orig;
1887                 alloc_size = phba->mem_req[i];
1888                 memset(mem_arr, 0, sizeof(struct mem_array) *
1889                        BEISCSI_MAX_FRAGS_INIT);
1890                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
1891                 do {
1892                         mem_arr->virtual_address = pci_alloc_consistent(
1893                                                         phba->pcidev,
1894                                                         curr_alloc_size,
1895                                                         &bus_add);
1896                         if (!mem_arr->virtual_address) {
1897                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
1898                                         goto free_mem;
1899                                 if (curr_alloc_size -
1900                                         rounddown_pow_of_two(curr_alloc_size))
1901                                         curr_alloc_size = rounddown_pow_of_two
1902                                                              (curr_alloc_size);
1903                                 else
1904                                         curr_alloc_size = curr_alloc_size / 2;
1905                         } else {
1906                                 mem_arr->bus_address.u.
1907                                     a64.address = (__u64) bus_add;
1908                                 mem_arr->size = curr_alloc_size;
1909                                 alloc_size -= curr_alloc_size;
1910                                 curr_alloc_size = min(be_max_phys_size *
1911                                                       1024, alloc_size);
1912                                 j++;
1913                                 mem_arr++;
1914                         }
1915                 } while (alloc_size);
1916                 mem_descr->num_elements = j;
1917                 mem_descr->size_in_bytes = phba->mem_req[i];
1918                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
1919                                                GFP_KERNEL);
1920                 if (!mem_descr->mem_array)
1921                         goto free_mem;
1922
1923                 memcpy(mem_descr->mem_array, mem_arr_orig,
1924                        sizeof(struct mem_array) * j);
1925                 mem_descr++;
1926         }
1927         kfree(mem_arr_orig);
1928         return 0;
1929 free_mem:
1930         mem_descr->num_elements = j;
1931         while ((i) || (j)) {
1932                 for (j = mem_descr->num_elements; j > 0; j--) {
1933                         pci_free_consistent(phba->pcidev,
1934                                             mem_descr->mem_array[j - 1].size,
1935                                             mem_descr->mem_array[j - 1].
1936                                             virtual_address,
1937                                             mem_descr->mem_array[j - 1].
1938                                             bus_address.u.a64.address);
1939                 }
1940                 if (i) {
1941                         i--;
1942                         kfree(mem_descr->mem_array);
1943                         mem_descr--;
1944                 }
1945         }
1946         kfree(mem_arr_orig);
1947         kfree(phba->init_mem);
1948         kfree(phba->phwi_ctrlr);
1949         return -ENOMEM;
1950 }
1951
1952 static int beiscsi_get_memory(struct beiscsi_hba *phba)
1953 {
1954         beiscsi_find_mem_req(phba);
1955         return beiscsi_alloc_mem(phba);
1956 }
1957
1958 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
1959 {
1960         struct pdu_data_out *pdata_out;
1961         struct pdu_nop_out *pnop_out;
1962         struct be_mem_descriptor *mem_descr;
1963
1964         mem_descr = phba->init_mem;
1965         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
1966         pdata_out =
1967             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
1968         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1969
1970         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
1971                       IIOC_SCSI_DATA);
1972
1973         pnop_out =
1974             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
1975                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
1976
1977         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1978         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
1979         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
1980         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
1981 }
1982
1983 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1984 {
1985         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
1986         struct wrb_handle *pwrb_handle;
1987         struct hwi_controller *phwi_ctrlr;
1988         struct hwi_wrb_context *pwrb_context;
1989         struct iscsi_wrb *pwrb;
1990         unsigned int num_cxn_wrbh;
1991         unsigned int num_cxn_wrb, j, idx, index;
1992
1993         mem_descr_wrbh = phba->init_mem;
1994         mem_descr_wrbh += HWI_MEM_WRBH;
1995
1996         mem_descr_wrb = phba->init_mem;
1997         mem_descr_wrb += HWI_MEM_WRB;
1998
1999         idx = 0;
2000         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2001         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2002                         ((sizeof(struct wrb_handle)) *
2003                          phba->params.wrbs_per_cxn));
2004         phwi_ctrlr = phba->phwi_ctrlr;
2005
2006         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2007                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2008                 pwrb_context->pwrb_handle_base =
2009                                 kzalloc(sizeof(struct wrb_handle *) *
2010                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2011                 pwrb_context->pwrb_handle_basestd =
2012                                 kzalloc(sizeof(struct wrb_handle *) *
2013                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2014                 if (num_cxn_wrbh) {
2015                         pwrb_context->alloc_index = 0;
2016                         pwrb_context->wrb_handles_available = 0;
2017                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2018                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2019                                 pwrb_context->pwrb_handle_basestd[j] =
2020                                                                 pwrb_handle;
2021                                 pwrb_context->wrb_handles_available++;
2022                                 pwrb_handle->wrb_index = j;
2023                                 pwrb_handle++;
2024                         }
2025                         pwrb_context->free_index = 0;
2026                         num_cxn_wrbh--;
2027                 } else {
2028                         idx++;
2029                         pwrb_handle =
2030                             mem_descr_wrbh->mem_array[idx].virtual_address;
2031                         num_cxn_wrbh =
2032                             ((mem_descr_wrbh->mem_array[idx].size) /
2033                              ((sizeof(struct wrb_handle)) *
2034                               phba->params.wrbs_per_cxn));
2035                         pwrb_context->alloc_index = 0;
2036                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2037                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2038                                 pwrb_context->pwrb_handle_basestd[j] =
2039                                     pwrb_handle;
2040                                 pwrb_context->wrb_handles_available++;
2041                                 pwrb_handle->wrb_index = j;
2042                                 pwrb_handle++;
2043                         }
2044                         pwrb_context->free_index = 0;
2045                         num_cxn_wrbh--;
2046                 }
2047         }
2048         idx = 0;
2049         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2050         num_cxn_wrb =
2051             ((mem_descr_wrb->mem_array[idx].size) / (sizeof(struct iscsi_wrb)) *
2052              phba->params.wrbs_per_cxn);
2053
2054         for (index = 0; index < phba->params.cxns_per_ctrl; index += 2) {
2055                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2056                 if (num_cxn_wrb) {
2057                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2058                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2059                                 pwrb_handle->pwrb = pwrb;
2060                                 pwrb++;
2061                         }
2062                         num_cxn_wrb--;
2063                 } else {
2064                         idx++;
2065                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2066                         num_cxn_wrb = ((mem_descr_wrb->mem_array[idx].size) /
2067                                         (sizeof(struct iscsi_wrb)) *
2068                                         phba->params.wrbs_per_cxn);
2069                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2070                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2071                                 pwrb_handle->pwrb = pwrb;
2072                                 pwrb++;
2073                         }
2074                         num_cxn_wrb--;
2075                 }
2076         }
2077 }
2078
2079 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2080 {
2081         struct hwi_controller *phwi_ctrlr;
2082         struct hba_parameters *p = &phba->params;
2083         struct hwi_async_pdu_context *pasync_ctx;
2084         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2085         unsigned int index;
2086         struct be_mem_descriptor *mem_descr;
2087
2088         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2089         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2090
2091         phwi_ctrlr = phba->phwi_ctrlr;
2092         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2093                                 mem_descr->mem_array[0].virtual_address;
2094         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2095         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2096
2097         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2098         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2099         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2100         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2101
2102         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2103         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2104         if (mem_descr->mem_array[0].virtual_address) {
2105                 SE_DEBUG(DBG_LVL_8,
2106                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2107                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2108         } else
2109                 shost_printk(KERN_WARNING, phba->shost,
2110                              "No Virtual address \n");
2111
2112         pasync_ctx->async_header.va_base =
2113                         mem_descr->mem_array[0].virtual_address;
2114
2115         pasync_ctx->async_header.pa_base.u.a64.address =
2116                         mem_descr->mem_array[0].bus_address.u.a64.address;
2117
2118         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2119         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2120         if (mem_descr->mem_array[0].virtual_address) {
2121                 SE_DEBUG(DBG_LVL_8,
2122                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2123                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2124         } else
2125                 shost_printk(KERN_WARNING, phba->shost,
2126                             "No Virtual address \n");
2127         pasync_ctx->async_header.ring_base =
2128                         mem_descr->mem_array[0].virtual_address;
2129
2130         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2131         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2132         if (mem_descr->mem_array[0].virtual_address) {
2133                 SE_DEBUG(DBG_LVL_8,
2134                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2135                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2136         } else
2137                 shost_printk(KERN_WARNING, phba->shost,
2138                             "No Virtual address \n");
2139
2140         pasync_ctx->async_header.handle_base =
2141                         mem_descr->mem_array[0].virtual_address;
2142         pasync_ctx->async_header.writables = 0;
2143         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2144
2145         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2146         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2147         if (mem_descr->mem_array[0].virtual_address) {
2148                 SE_DEBUG(DBG_LVL_8,
2149                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2150                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2151         } else
2152                 shost_printk(KERN_WARNING, phba->shost,
2153                             "No Virtual address \n");
2154         pasync_ctx->async_data.va_base =
2155                         mem_descr->mem_array[0].virtual_address;
2156         pasync_ctx->async_data.pa_base.u.a64.address =
2157                         mem_descr->mem_array[0].bus_address.u.a64.address;
2158
2159         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2160         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2161         if (mem_descr->mem_array[0].virtual_address) {
2162                 SE_DEBUG(DBG_LVL_8,
2163                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2164                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2165         } else
2166                 shost_printk(KERN_WARNING, phba->shost,
2167                              "No Virtual address \n");
2168
2169         pasync_ctx->async_data.ring_base =
2170                         mem_descr->mem_array[0].virtual_address;
2171
2172         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2173         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2174         if (!mem_descr->mem_array[0].virtual_address)
2175                 shost_printk(KERN_WARNING, phba->shost,
2176                             "No Virtual address \n");
2177
2178         pasync_ctx->async_data.handle_base =
2179                         mem_descr->mem_array[0].virtual_address;
2180         pasync_ctx->async_data.writables = 0;
2181         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2182
2183         pasync_header_h =
2184                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2185         pasync_data_h =
2186                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2187
2188         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2189                 pasync_header_h->cri = -1;
2190                 pasync_header_h->index = (char)index;
2191                 INIT_LIST_HEAD(&pasync_header_h->link);
2192                 pasync_header_h->pbuffer =
2193                         (void *)((unsigned long)
2194                         (pasync_ctx->async_header.va_base) +
2195                         (p->defpdu_hdr_sz * index));
2196
2197                 pasync_header_h->pa.u.a64.address =
2198                         pasync_ctx->async_header.pa_base.u.a64.address +
2199                         (p->defpdu_hdr_sz * index);
2200
2201                 list_add_tail(&pasync_header_h->link,
2202                                 &pasync_ctx->async_header.free_list);
2203                 pasync_header_h++;
2204                 pasync_ctx->async_header.free_entries++;
2205                 pasync_ctx->async_header.writables++;
2206
2207                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2208                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2209                                header_busy_list);
2210                 pasync_data_h->cri = -1;
2211                 pasync_data_h->index = (char)index;
2212                 INIT_LIST_HEAD(&pasync_data_h->link);
2213                 pasync_data_h->pbuffer =
2214                         (void *)((unsigned long)
2215                         (pasync_ctx->async_data.va_base) +
2216                         (p->defpdu_data_sz * index));
2217
2218                 pasync_data_h->pa.u.a64.address =
2219                     pasync_ctx->async_data.pa_base.u.a64.address +
2220                     (p->defpdu_data_sz * index);
2221
2222                 list_add_tail(&pasync_data_h->link,
2223                               &pasync_ctx->async_data.free_list);
2224                 pasync_data_h++;
2225                 pasync_ctx->async_data.free_entries++;
2226                 pasync_ctx->async_data.writables++;
2227
2228                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2229         }
2230
2231         pasync_ctx->async_header.host_write_ptr = 0;
2232         pasync_ctx->async_header.ep_read_ptr = -1;
2233         pasync_ctx->async_data.host_write_ptr = 0;
2234         pasync_ctx->async_data.ep_read_ptr = -1;
2235 }
2236
2237 static int
2238 be_sgl_create_contiguous(void *virtual_address,
2239                          u64 physical_address, u32 length,
2240                          struct be_dma_mem *sgl)
2241 {
2242         WARN_ON(!virtual_address);
2243         WARN_ON(!physical_address);
2244         WARN_ON(!length > 0);
2245         WARN_ON(!sgl);
2246
2247         sgl->va = virtual_address;
2248         sgl->dma = physical_address;
2249         sgl->size = length;
2250
2251         return 0;
2252 }
2253
2254 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2255 {
2256         memset(sgl, 0, sizeof(*sgl));
2257 }
2258
2259 static void
2260 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2261                      struct mem_array *pmem, struct be_dma_mem *sgl)
2262 {
2263         if (sgl->va)
2264                 be_sgl_destroy_contiguous(sgl);
2265
2266         be_sgl_create_contiguous(pmem->virtual_address,
2267                                  pmem->bus_address.u.a64.address,
2268                                  pmem->size, sgl);
2269 }
2270
2271 static void
2272 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2273                            struct mem_array *pmem, struct be_dma_mem *sgl)
2274 {
2275         if (sgl->va)
2276                 be_sgl_destroy_contiguous(sgl);
2277
2278         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2279                                  pmem->bus_address.u.a64.address,
2280                                  pmem->size, sgl);
2281 }
2282
2283 static int be_fill_queue(struct be_queue_info *q,
2284                 u16 len, u16 entry_size, void *vaddress)
2285 {
2286         struct be_dma_mem *mem = &q->dma_mem;
2287
2288         memset(q, 0, sizeof(*q));
2289         q->len = len;
2290         q->entry_size = entry_size;
2291         mem->size = len * entry_size;
2292         mem->va = vaddress;
2293         if (!mem->va)
2294                 return -ENOMEM;
2295         memset(mem->va, 0, mem->size);
2296         return 0;
2297 }
2298
2299 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2300                              struct hwi_context_memory *phwi_context)
2301 {
2302         unsigned int i, num_eq_pages;
2303         int ret, eq_for_mcc;
2304         struct be_queue_info *eq;
2305         struct be_dma_mem *mem;
2306         void *eq_vaddress;
2307         dma_addr_t paddr;
2308
2309         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2310                                       sizeof(struct be_eq_entry));
2311
2312         if (phba->msix_enabled)
2313                 eq_for_mcc = 1;
2314         else
2315                 eq_for_mcc = 0;
2316         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2317                 eq = &phwi_context->be_eq[i].q;
2318                 mem = &eq->dma_mem;
2319                 phwi_context->be_eq[i].phba = phba;
2320                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2321                                                      num_eq_pages * PAGE_SIZE,
2322                                                      &paddr);
2323                 if (!eq_vaddress)
2324                         goto create_eq_error;
2325
2326                 mem->va = eq_vaddress;
2327                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2328                                     sizeof(struct be_eq_entry), eq_vaddress);
2329                 if (ret) {
2330                         shost_printk(KERN_ERR, phba->shost,
2331                                      "be_fill_queue Failed for EQ \n");
2332                         goto create_eq_error;
2333                 }
2334
2335                 mem->dma = paddr;
2336                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2337                                             phwi_context->cur_eqd);
2338                 if (ret) {
2339                         shost_printk(KERN_ERR, phba->shost,
2340                                      "beiscsi_cmd_eq_create"
2341                                      "Failedfor EQ \n");
2342                         goto create_eq_error;
2343                 }
2344                 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2345         }
2346         return 0;
2347 create_eq_error:
2348         for (i = 0; i < (phba->num_cpus + 1); i++) {
2349                 eq = &phwi_context->be_eq[i].q;
2350                 mem = &eq->dma_mem;
2351                 if (mem->va)
2352                         pci_free_consistent(phba->pcidev, num_eq_pages
2353                                             * PAGE_SIZE,
2354                                             mem->va, mem->dma);
2355         }
2356         return ret;
2357 }
2358
2359 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2360                              struct hwi_context_memory *phwi_context)
2361 {
2362         unsigned int i, num_cq_pages;
2363         int ret;
2364         struct be_queue_info *cq, *eq;
2365         struct be_dma_mem *mem;
2366         struct be_eq_obj *pbe_eq;
2367         void *cq_vaddress;
2368         dma_addr_t paddr;
2369
2370         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2371                                       sizeof(struct sol_cqe));
2372
2373         for (i = 0; i < phba->num_cpus; i++) {
2374                 cq = &phwi_context->be_cq[i];
2375                 eq = &phwi_context->be_eq[i].q;
2376                 pbe_eq = &phwi_context->be_eq[i];
2377                 pbe_eq->cq = cq;
2378                 pbe_eq->phba = phba;
2379                 mem = &cq->dma_mem;
2380                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2381                                                      num_cq_pages * PAGE_SIZE,
2382                                                      &paddr);
2383                 if (!cq_vaddress)
2384                         goto create_cq_error;
2385                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2386                                     sizeof(struct sol_cqe), cq_vaddress);
2387                 if (ret) {
2388                         shost_printk(KERN_ERR, phba->shost,
2389                                      "be_fill_queue Failed for ISCSI CQ \n");
2390                         goto create_cq_error;
2391                 }
2392
2393                 mem->dma = paddr;
2394                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2395                                             false, 0);
2396                 if (ret) {
2397                         shost_printk(KERN_ERR, phba->shost,
2398                                      "beiscsi_cmd_eq_create"
2399                                      "Failed for ISCSI CQ \n");
2400                         goto create_cq_error;
2401                 }
2402                 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2403                                                  cq->id, eq->id);
2404                 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2405         }
2406         return 0;
2407
2408 create_cq_error:
2409         for (i = 0; i < phba->num_cpus; i++) {
2410                 cq = &phwi_context->be_cq[i];
2411                 mem = &cq->dma_mem;
2412                 if (mem->va)
2413                         pci_free_consistent(phba->pcidev, num_cq_pages
2414                                             * PAGE_SIZE,
2415                                             mem->va, mem->dma);
2416         }
2417         return ret;
2418
2419 }
2420
2421 static int
2422 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2423                        struct hwi_context_memory *phwi_context,
2424                        struct hwi_controller *phwi_ctrlr,
2425                        unsigned int def_pdu_ring_sz)
2426 {
2427         unsigned int idx;
2428         int ret;
2429         struct be_queue_info *dq, *cq;
2430         struct be_dma_mem *mem;
2431         struct be_mem_descriptor *mem_descr;
2432         void *dq_vaddress;
2433
2434         idx = 0;
2435         dq = &phwi_context->be_def_hdrq;
2436         cq = &phwi_context->be_cq[0];
2437         mem = &dq->dma_mem;
2438         mem_descr = phba->init_mem;
2439         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2440         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2441         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2442                             sizeof(struct phys_addr),
2443                             sizeof(struct phys_addr), dq_vaddress);
2444         if (ret) {
2445                 shost_printk(KERN_ERR, phba->shost,
2446                              "be_fill_queue Failed for DEF PDU HDR\n");
2447                 return ret;
2448         }
2449         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2450         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2451                                               def_pdu_ring_sz,
2452                                               phba->params.defpdu_hdr_sz);
2453         if (ret) {
2454                 shost_printk(KERN_ERR, phba->shost,
2455                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2456                 return ret;
2457         }
2458         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2459         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2460                  phwi_context->be_def_hdrq.id);
2461         hwi_post_async_buffers(phba, 1);
2462         return 0;
2463 }
2464
2465 static int
2466 beiscsi_create_def_data(struct beiscsi_hba *phba,
2467                         struct hwi_context_memory *phwi_context,
2468                         struct hwi_controller *phwi_ctrlr,
2469                         unsigned int def_pdu_ring_sz)
2470 {
2471         unsigned int idx;
2472         int ret;
2473         struct be_queue_info *dataq, *cq;
2474         struct be_dma_mem *mem;
2475         struct be_mem_descriptor *mem_descr;
2476         void *dq_vaddress;
2477
2478         idx = 0;
2479         dataq = &phwi_context->be_def_dataq;
2480         cq = &phwi_context->be_cq[0];
2481         mem = &dataq->dma_mem;
2482         mem_descr = phba->init_mem;
2483         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2484         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2485         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2486                             sizeof(struct phys_addr),
2487                             sizeof(struct phys_addr), dq_vaddress);
2488         if (ret) {
2489                 shost_printk(KERN_ERR, phba->shost,
2490                              "be_fill_queue Failed for DEF PDU DATA\n");
2491                 return ret;
2492         }
2493         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2494         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2495                                               def_pdu_ring_sz,
2496                                               phba->params.defpdu_data_sz);
2497         if (ret) {
2498                 shost_printk(KERN_ERR, phba->shost,
2499                              "be_cmd_create_default_pdu_queue Failed"
2500                              " for DEF PDU DATA\n");
2501                 return ret;
2502         }
2503         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2504         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2505                  phwi_context->be_def_dataq.id);
2506         hwi_post_async_buffers(phba, 0);
2507         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2508         return 0;
2509 }
2510
2511 static int
2512 beiscsi_post_pages(struct beiscsi_hba *phba)
2513 {
2514         struct be_mem_descriptor *mem_descr;
2515         struct mem_array *pm_arr;
2516         unsigned int page_offset, i;
2517         struct be_dma_mem sgl;
2518         int status;
2519
2520         mem_descr = phba->init_mem;
2521         mem_descr += HWI_MEM_SGE;
2522         pm_arr = mem_descr->mem_array;
2523
2524         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2525                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2526         for (i = 0; i < mem_descr->num_elements; i++) {
2527                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2528                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2529                                                 page_offset,
2530                                                 (pm_arr->size / PAGE_SIZE));
2531                 page_offset += pm_arr->size / PAGE_SIZE;
2532                 if (status != 0) {
2533                         shost_printk(KERN_ERR, phba->shost,
2534                                      "post sgl failed.\n");
2535                         return status;
2536                 }
2537                 pm_arr++;
2538         }
2539         SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2540         return 0;
2541 }
2542
2543 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2544 {
2545         struct be_dma_mem *mem = &q->dma_mem;
2546         if (mem->va)
2547                 pci_free_consistent(phba->pcidev, mem->size,
2548                         mem->va, mem->dma);
2549 }
2550
2551 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2552                 u16 len, u16 entry_size)
2553 {
2554         struct be_dma_mem *mem = &q->dma_mem;
2555
2556         memset(q, 0, sizeof(*q));
2557         q->len = len;
2558         q->entry_size = entry_size;
2559         mem->size = len * entry_size;
2560         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2561         if (!mem->va)
2562                 return -1;
2563         memset(mem->va, 0, mem->size);
2564         return 0;
2565 }
2566
2567 static int
2568 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2569                          struct hwi_context_memory *phwi_context,
2570                          struct hwi_controller *phwi_ctrlr)
2571 {
2572         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2573         u64 pa_addr_lo;
2574         unsigned int idx, num, i;
2575         struct mem_array *pwrb_arr;
2576         void *wrb_vaddr;
2577         struct be_dma_mem sgl;
2578         struct be_mem_descriptor *mem_descr;
2579         int status;
2580
2581         idx = 0;
2582         mem_descr = phba->init_mem;
2583         mem_descr += HWI_MEM_WRB;
2584         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2585                            GFP_KERNEL);
2586         if (!pwrb_arr) {
2587                 shost_printk(KERN_ERR, phba->shost,
2588                              "Memory alloc failed in create wrb ring.\n");
2589                 return -ENOMEM;
2590         }
2591         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2592         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2593         num_wrb_rings = mem_descr->mem_array[idx].size /
2594                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2595
2596         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2597                 if (num_wrb_rings) {
2598                         pwrb_arr[num].virtual_address = wrb_vaddr;
2599                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2600                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2601                                             sizeof(struct iscsi_wrb);
2602                         wrb_vaddr += pwrb_arr[num].size;
2603                         pa_addr_lo += pwrb_arr[num].size;
2604                         num_wrb_rings--;
2605                 } else {
2606                         idx++;
2607                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2608                         pa_addr_lo = mem_descr->mem_array[idx].\
2609                                         bus_address.u.a64.address;
2610                         num_wrb_rings = mem_descr->mem_array[idx].size /
2611                                         (phba->params.wrbs_per_cxn *
2612                                         sizeof(struct iscsi_wrb));
2613                         pwrb_arr[num].virtual_address = wrb_vaddr;
2614                         pwrb_arr[num].bus_address.u.a64.address\
2615                                                 = pa_addr_lo;
2616                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2617                                                  sizeof(struct iscsi_wrb);
2618                         wrb_vaddr += pwrb_arr[num].size;
2619                         pa_addr_lo   += pwrb_arr[num].size;
2620                         num_wrb_rings--;
2621                 }
2622         }
2623         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2624                 wrb_mem_index = 0;
2625                 offset = 0;
2626                 size = 0;
2627
2628                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2629                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2630                                             &phwi_context->be_wrbq[i]);
2631                 if (status != 0) {
2632                         shost_printk(KERN_ERR, phba->shost,
2633                                      "wrbq create failed.");
2634                         return status;
2635                 }
2636                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
2637                                                                    id;
2638         }
2639         kfree(pwrb_arr);
2640         return 0;
2641 }
2642
2643 static void free_wrb_handles(struct beiscsi_hba *phba)
2644 {
2645         unsigned int index;
2646         struct hwi_controller *phwi_ctrlr;
2647         struct hwi_wrb_context *pwrb_context;
2648
2649         phwi_ctrlr = phba->phwi_ctrlr;
2650         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2651                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2652                 kfree(pwrb_context->pwrb_handle_base);
2653                 kfree(pwrb_context->pwrb_handle_basestd);
2654         }
2655 }
2656
2657 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
2658 {
2659         struct be_queue_info *q;
2660         struct be_ctrl_info *ctrl = &phba->ctrl;
2661
2662         q = &phba->ctrl.mcc_obj.q;
2663         if (q->created)
2664                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
2665         be_queue_free(phba, q);
2666
2667         q = &phba->ctrl.mcc_obj.cq;
2668         if (q->created)
2669                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2670         be_queue_free(phba, q);
2671 }
2672
2673 static void hwi_cleanup(struct beiscsi_hba *phba)
2674 {
2675         struct be_queue_info *q;
2676         struct be_ctrl_info *ctrl = &phba->ctrl;
2677         struct hwi_controller *phwi_ctrlr;
2678         struct hwi_context_memory *phwi_context;
2679         int i, eq_num;
2680
2681         phwi_ctrlr = phba->phwi_ctrlr;
2682         phwi_context = phwi_ctrlr->phwi_ctxt;
2683         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2684                 q = &phwi_context->be_wrbq[i];
2685                 if (q->created)
2686                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2687         }
2688         free_wrb_handles(phba);
2689
2690         q = &phwi_context->be_def_hdrq;
2691         if (q->created)
2692                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2693
2694         q = &phwi_context->be_def_dataq;
2695         if (q->created)
2696                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2697
2698         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2699
2700         for (i = 0; i < (phba->num_cpus); i++) {
2701                 q = &phwi_context->be_cq[i];
2702                 if (q->created)
2703                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2704         }
2705         if (phba->msix_enabled)
2706                 eq_num = 1;
2707         else
2708                 eq_num = 0;
2709         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
2710                 q = &phwi_context->be_eq[i].q;
2711                 if (q->created)
2712                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2713         }
2714         be_mcc_queues_destroy(phba);
2715 }
2716
2717 static int be_mcc_queues_create(struct beiscsi_hba *phba,
2718                                 struct hwi_context_memory *phwi_context)
2719 {
2720         struct be_queue_info *q, *cq;
2721         struct be_ctrl_info *ctrl = &phba->ctrl;
2722
2723         /* Alloc MCC compl queue */
2724         cq = &phba->ctrl.mcc_obj.cq;
2725         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
2726                         sizeof(struct be_mcc_compl)))
2727                 goto err;
2728         /* Ask BE to create MCC compl queue; */
2729         if (phba->msix_enabled) {
2730                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
2731                                          [phba->num_cpus].q, false, true, 0))
2732                 goto mcc_cq_free;
2733         } else {
2734                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
2735                                           false, true, 0))
2736                 goto mcc_cq_free;
2737         }
2738
2739         /* Alloc MCC queue */
2740         q = &phba->ctrl.mcc_obj.q;
2741         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2742                 goto mcc_cq_destroy;
2743
2744         /* Ask BE to create MCC queue */
2745         if (beiscsi_cmd_mccq_create(phba, q, cq))
2746                 goto mcc_q_free;
2747
2748         return 0;
2749
2750 mcc_q_free:
2751         be_queue_free(phba, q);
2752 mcc_cq_destroy:
2753         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
2754 mcc_cq_free:
2755         be_queue_free(phba, cq);
2756 err:
2757         return -1;
2758 }
2759
2760 static int find_num_cpus(void)
2761 {
2762         int  num_cpus = 0;
2763
2764         num_cpus = num_online_cpus();
2765         if (num_cpus >= MAX_CPUS)
2766                 num_cpus = MAX_CPUS - 1;
2767
2768         SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", num_cpus);
2769         return num_cpus;
2770 }
2771
2772 static int hwi_init_port(struct beiscsi_hba *phba)
2773 {
2774         struct hwi_controller *phwi_ctrlr;
2775         struct hwi_context_memory *phwi_context;
2776         unsigned int def_pdu_ring_sz;
2777         struct be_ctrl_info *ctrl = &phba->ctrl;
2778         int status;
2779
2780         def_pdu_ring_sz =
2781                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2782         phwi_ctrlr = phba->phwi_ctrlr;
2783         phwi_context = phwi_ctrlr->phwi_ctxt;
2784         phwi_context->max_eqd = 0;
2785         phwi_context->min_eqd = 0;
2786         phwi_context->cur_eqd = 64;
2787         be_cmd_fw_initialize(&phba->ctrl);
2788
2789         status = beiscsi_create_eqs(phba, phwi_context);
2790         if (status != 0) {
2791                 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2792                 goto error;
2793         }
2794
2795         status = be_mcc_queues_create(phba, phwi_context);
2796         if (status != 0)
2797                 goto error;
2798
2799         status = mgmt_check_supported_fw(ctrl, phba);
2800         if (status != 0) {
2801                 shost_printk(KERN_ERR, phba->shost,
2802                              "Unsupported fw version \n");
2803                 goto error;
2804         }
2805
2806         if (phba->fw_config.iscsi_features == 0x1)
2807                 ring_mode = 1;
2808         else
2809                 ring_mode = 0;
2810
2811         status = beiscsi_create_cqs(phba, phwi_context);
2812         if (status != 0) {
2813                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2814                 goto error;
2815         }
2816
2817         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2818                                         def_pdu_ring_sz);
2819         if (status != 0) {
2820                 shost_printk(KERN_ERR, phba->shost,
2821                              "Default Header not created\n");
2822                 goto error;
2823         }
2824
2825         status = beiscsi_create_def_data(phba, phwi_context,
2826                                          phwi_ctrlr, def_pdu_ring_sz);
2827         if (status != 0) {
2828                 shost_printk(KERN_ERR, phba->shost,
2829                              "Default Data not created\n");
2830                 goto error;
2831         }
2832
2833         status = beiscsi_post_pages(phba);
2834         if (status != 0) {
2835                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2836                 goto error;
2837         }
2838
2839         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2840         if (status != 0) {
2841                 shost_printk(KERN_ERR, phba->shost,
2842                              "WRB Rings not created\n");
2843                 goto error;
2844         }
2845
2846         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2847         return 0;
2848
2849 error:
2850         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2851         hwi_cleanup(phba);
2852         return -ENOMEM;
2853 }
2854
2855 static int hwi_init_controller(struct beiscsi_hba *phba)
2856 {
2857         struct hwi_controller *phwi_ctrlr;
2858
2859         phwi_ctrlr = phba->phwi_ctrlr;
2860         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2861                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2862                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2863                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2864                          phwi_ctrlr->phwi_ctxt);
2865         } else {
2866                 shost_printk(KERN_ERR, phba->shost,
2867                              "HWI_MEM_ADDN_CONTEXT is more than one element."
2868                              "Failing to load\n");
2869                 return -ENOMEM;
2870         }
2871
2872         iscsi_init_global_templates(phba);
2873         beiscsi_init_wrb_handle(phba);
2874         hwi_init_async_pdu_ctx(phba);
2875         if (hwi_init_port(phba) != 0) {
2876                 shost_printk(KERN_ERR, phba->shost,
2877                              "hwi_init_controller failed\n");
2878                 return -ENOMEM;
2879         }
2880         return 0;
2881 }
2882
2883 static void beiscsi_free_mem(struct beiscsi_hba *phba)
2884 {
2885         struct be_mem_descriptor *mem_descr;
2886         int i, j;
2887
2888         mem_descr = phba->init_mem;
2889         i = 0;
2890         j = 0;
2891         for (i = 0; i < SE_MEM_MAX; i++) {
2892                 for (j = mem_descr->num_elements; j > 0; j--) {
2893                         pci_free_consistent(phba->pcidev,
2894                           mem_descr->mem_array[j - 1].size,
2895                           mem_descr->mem_array[j - 1].virtual_address,
2896                           mem_descr->mem_array[j - 1].bus_address.
2897                                 u.a64.address);
2898                 }
2899                 kfree(mem_descr->mem_array);
2900                 mem_descr++;
2901         }
2902         kfree(phba->init_mem);
2903         kfree(phba->phwi_ctrlr);
2904 }
2905
2906 static int beiscsi_init_controller(struct beiscsi_hba *phba)
2907 {
2908         int ret = -ENOMEM;
2909
2910         ret = beiscsi_get_memory(phba);
2911         if (ret < 0) {
2912                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
2913                              "Failed in beiscsi_alloc_memory \n");
2914                 return ret;
2915         }
2916
2917         ret = hwi_init_controller(phba);
2918         if (ret)
2919                 goto free_init;
2920         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
2921         return 0;
2922
2923 free_init:
2924         beiscsi_free_mem(phba);
2925         return -ENOMEM;
2926 }
2927
2928 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
2929 {
2930         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
2931         struct sgl_handle *psgl_handle;
2932         struct iscsi_sge *pfrag;
2933         unsigned int arr_index, i, idx;
2934
2935         phba->io_sgl_hndl_avbl = 0;
2936         phba->eh_sgl_hndl_avbl = 0;
2937
2938         if (ring_mode) {
2939                 phba->sgl_hndl_array = kzalloc(sizeof(struct sgl_handle *) *
2940                                               phba->params.icds_per_ctrl,
2941                                                  GFP_KERNEL);
2942                 if (!phba->sgl_hndl_array) {
2943                         shost_printk(KERN_ERR, phba->shost,
2944                              "Mem Alloc Failed. Failing to load\n");
2945                         return -ENOMEM;
2946                 }
2947         }
2948
2949         mem_descr_sglh = phba->init_mem;
2950         mem_descr_sglh += HWI_MEM_SGLH;
2951         if (1 == mem_descr_sglh->num_elements) {
2952                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2953                                                  phba->params.ios_per_ctrl,
2954                                                  GFP_KERNEL);
2955                 if (!phba->io_sgl_hndl_base) {
2956                         if (ring_mode)
2957                                 kfree(phba->sgl_hndl_array);
2958                         shost_printk(KERN_ERR, phba->shost,
2959                                      "Mem Alloc Failed. Failing to load\n");
2960                         return -ENOMEM;
2961                 }
2962                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2963                                                  (phba->params.icds_per_ctrl -
2964                                                  phba->params.ios_per_ctrl),
2965                                                  GFP_KERNEL);
2966                 if (!phba->eh_sgl_hndl_base) {
2967                         kfree(phba->io_sgl_hndl_base);
2968                         shost_printk(KERN_ERR, phba->shost,
2969                                      "Mem Alloc Failed. Failing to load\n");
2970                         return -ENOMEM;
2971                 }
2972         } else {
2973                 shost_printk(KERN_ERR, phba->shost,
2974                              "HWI_MEM_SGLH is more than one element."
2975                              "Failing to load\n");
2976                 return -ENOMEM;
2977         }
2978
2979         arr_index = 0;
2980         idx = 0;
2981         while (idx < mem_descr_sglh->num_elements) {
2982                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
2983
2984                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
2985                       sizeof(struct sgl_handle)); i++) {
2986                         if (arr_index < phba->params.ios_per_ctrl) {
2987                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
2988                                 phba->io_sgl_hndl_avbl++;
2989                                 arr_index++;
2990                         } else {
2991                                 phba->eh_sgl_hndl_base[arr_index -
2992                                         phba->params.ios_per_ctrl] =
2993                                                                 psgl_handle;
2994                                 arr_index++;
2995                                 phba->eh_sgl_hndl_avbl++;
2996                         }
2997                         psgl_handle++;
2998                 }
2999                 idx++;
3000         }
3001         SE_DEBUG(DBG_LVL_8,
3002                  "phba->io_sgl_hndl_avbl=%d"
3003                  "phba->eh_sgl_hndl_avbl=%d \n",
3004                  phba->io_sgl_hndl_avbl,
3005                  phba->eh_sgl_hndl_avbl);
3006         mem_descr_sg = phba->init_mem;
3007         mem_descr_sg += HWI_MEM_SGE;
3008         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
3009                  mem_descr_sg->num_elements);
3010         arr_index = 0;
3011         idx = 0;
3012         while (idx < mem_descr_sg->num_elements) {
3013                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3014
3015                 for (i = 0;
3016                      i < (mem_descr_sg->mem_array[idx].size) /
3017                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3018                      i++) {
3019                         if (arr_index < phba->params.ios_per_ctrl)
3020                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3021                         else
3022                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3023                                                 phba->params.ios_per_ctrl];
3024                         psgl_handle->pfrag = pfrag;
3025                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3026                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3027                         pfrag += phba->params.num_sge_per_io;
3028                         psgl_handle->sgl_index =
3029                                 phba->fw_config.iscsi_icd_start + arr_index++;
3030                 }
3031                 idx++;
3032         }
3033         phba->io_sgl_free_index = 0;
3034         phba->io_sgl_alloc_index = 0;
3035         phba->eh_sgl_free_index = 0;
3036         phba->eh_sgl_alloc_index = 0;
3037         return 0;
3038 }
3039
3040 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3041 {
3042         int i, new_cid;
3043
3044         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3045                                   GFP_KERNEL);
3046         if (!phba->cid_array) {
3047                 shost_printk(KERN_ERR, phba->shost,
3048                              "Failed to allocate memory in "
3049                              "hba_setup_cid_tbls\n");
3050                 return -ENOMEM;
3051         }
3052         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3053                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3054         if (!phba->ep_array) {
3055                 shost_printk(KERN_ERR, phba->shost,
3056                              "Failed to allocate memory in "
3057                              "hba_setup_cid_tbls \n");
3058                 kfree(phba->cid_array);
3059                 return -ENOMEM;
3060         }
3061         new_cid = phba->fw_config.iscsi_cid_start;
3062         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3063                 phba->cid_array[i] = new_cid;
3064                 new_cid += 2;
3065         }
3066         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3067         return 0;
3068 }
3069
3070 static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
3071 {
3072         struct be_ctrl_info *ctrl = &phba->ctrl;
3073         struct hwi_controller *phwi_ctrlr;
3074         struct hwi_context_memory *phwi_context;
3075         struct be_queue_info *eq;
3076         u8 __iomem *addr;
3077         u32 reg, i;
3078         u32 enabled;
3079
3080         phwi_ctrlr = phba->phwi_ctrlr;
3081         phwi_context = phwi_ctrlr->phwi_ctxt;
3082
3083         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3084                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3085         reg = ioread32(addr);
3086         SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
3087
3088         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3089         if (!enabled) {
3090                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3091                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
3092                 iowrite32(reg, addr);
3093                 for (i = 0; i <= phba->num_cpus; i++) {
3094                         eq = &phwi_context->be_eq[i].q;
3095                         SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3096                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3097                 }
3098         } else
3099                 shost_printk(KERN_WARNING, phba->shost,
3100                              "In hwi_enable_intr, Not Enabled \n");
3101         return true;
3102 }
3103
3104 static void hwi_disable_intr(struct beiscsi_hba *phba)
3105 {
3106         struct be_ctrl_info *ctrl = &phba->ctrl;
3107
3108         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3109         u32 reg = ioread32(addr);
3110
3111         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3112         if (enabled) {
3113                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3114                 iowrite32(reg, addr);
3115         } else
3116                 shost_printk(KERN_WARNING, phba->shost,
3117                              "In hwi_disable_intr, Already Disabled \n");
3118 }
3119
3120 static int beiscsi_init_port(struct beiscsi_hba *phba)
3121 {
3122         int ret;
3123
3124         ret = beiscsi_init_controller(phba);
3125         if (ret < 0) {
3126                 shost_printk(KERN_ERR, phba->shost,
3127                              "beiscsi_dev_probe - Failed in"
3128                              "beiscsi_init_controller \n");
3129                 return ret;
3130         }
3131         ret = beiscsi_init_sgl_handle(phba);
3132         if (ret < 0) {
3133                 shost_printk(KERN_ERR, phba->shost,
3134                              "beiscsi_dev_probe - Failed in"
3135                              "beiscsi_init_sgl_handle \n");
3136                 goto do_cleanup_ctrlr;
3137         }
3138
3139         if (hba_setup_cid_tbls(phba)) {
3140                 shost_printk(KERN_ERR, phba->shost,
3141                              "Failed in hba_setup_cid_tbls\n");
3142                 if (ring_mode)
3143                         kfree(phba->sgl_hndl_array);
3144                 kfree(phba->io_sgl_hndl_base);
3145                 kfree(phba->eh_sgl_hndl_base);
3146                 goto do_cleanup_ctrlr;
3147         }
3148
3149         return ret;
3150
3151 do_cleanup_ctrlr:
3152         hwi_cleanup(phba);
3153         return ret;
3154 }
3155
3156 static void hwi_purge_eq(struct beiscsi_hba *phba)
3157 {
3158         struct hwi_controller *phwi_ctrlr;
3159         struct hwi_context_memory *phwi_context;
3160         struct be_queue_info *eq;
3161         struct be_eq_entry *eqe = NULL;
3162         int i, eq_msix;
3163
3164         phwi_ctrlr = phba->phwi_ctrlr;
3165         phwi_context = phwi_ctrlr->phwi_ctxt;
3166         if (phba->msix_enabled)
3167                 eq_msix = 1;
3168         else
3169                 eq_msix = 0;
3170
3171         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3172                 eq = &phwi_context->be_eq[i].q;
3173                 eqe = queue_tail_node(eq);
3174
3175                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3176                                         & EQE_VALID_MASK) {
3177                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3178                         queue_tail_inc(eq);
3179                         eqe = queue_tail_node(eq);
3180                 }
3181         }
3182 }
3183
3184 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3185 {
3186         unsigned char mgmt_status;
3187
3188         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3189         if (mgmt_status)
3190                 shost_printk(KERN_WARNING, phba->shost,
3191                              "mgmt_epfw_cleanup FAILED \n");
3192         hwi_cleanup(phba);
3193         hwi_purge_eq(phba);
3194         if (ring_mode)
3195                 kfree(phba->sgl_hndl_array);
3196         kfree(phba->io_sgl_hndl_base);
3197         kfree(phba->eh_sgl_hndl_base);
3198         kfree(phba->cid_array);
3199         kfree(phba->ep_array);
3200 }
3201
3202 void
3203 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3204                            struct beiscsi_offload_params *params)
3205 {
3206         struct wrb_handle *pwrb_handle;
3207         struct iscsi_target_context_update_wrb *pwrb = NULL;
3208         struct be_mem_descriptor *mem_descr;
3209         struct beiscsi_hba *phba = beiscsi_conn->phba;
3210         u32 doorbell = 0;
3211
3212         /*
3213          * We can always use 0 here because it is reserved by libiscsi for
3214          * login/startup related tasks.
3215          */
3216         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3217                                        phba->fw_config.iscsi_cid_start));
3218         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3219         memset(pwrb, 0, sizeof(*pwrb));
3220         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3221                       max_burst_length, pwrb, params->dw[offsetof
3222                       (struct amap_beiscsi_offload_params,
3223                       max_burst_length) / 32]);
3224         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3225                       max_send_data_segment_length, pwrb,
3226                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3227                       max_send_data_segment_length) / 32]);
3228         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3229                       first_burst_length,
3230                       pwrb,
3231                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3232                       first_burst_length) / 32]);
3233
3234         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3235                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3236                       erl) / 32] & OFFLD_PARAMS_ERL));
3237         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3238                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3239                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3240         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3241                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3242                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3243         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3244                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3245                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3246         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3247                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3248                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3249         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3250                       pwrb,
3251                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3252                       exp_statsn) / 32] + 1));
3253         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3254                       0x7);
3255         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3256                       pwrb, pwrb_handle->wrb_index);
3257         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3258                       pwrb, pwrb_handle->nxt_wrb_index);
3259         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3260                         session_state, pwrb, 0);
3261         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3262                       pwrb, 1);
3263         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3264                       pwrb, 0);
3265         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3266                       0);
3267
3268         mem_descr = phba->init_mem;
3269         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3270
3271         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3272                         pad_buffer_addr_hi, pwrb,
3273                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3274         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3275                         pad_buffer_addr_lo, pwrb,
3276                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3277
3278         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3279
3280         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3281         if (!ring_mode)
3282                 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3283                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
3284         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3285
3286         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3287 }
3288
3289 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3290                               int *index, int *age)
3291 {
3292         *index = (int)itt;
3293         if (age)
3294                 *age = conn->session->age;
3295 }
3296
3297 /**
3298  * beiscsi_alloc_pdu - allocates pdu and related resources
3299  * @task: libiscsi task
3300  * @opcode: opcode of pdu for task
3301  *
3302  * This is called with the session lock held. It will allocate
3303  * the wrb and sgl if needed for the command. And it will prep
3304  * the pdu's itt. beiscsi_parse_pdu will later translate
3305  * the pdu itt to the libiscsi task itt.
3306  */
3307 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3308 {
3309         struct beiscsi_io_task *io_task = task->dd_data;
3310         struct iscsi_conn *conn = task->conn;
3311         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3312         struct beiscsi_hba *phba = beiscsi_conn->phba;
3313         struct hwi_wrb_context *pwrb_context;
3314         struct hwi_controller *phwi_ctrlr;
3315         itt_t itt;
3316         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3317         dma_addr_t paddr;
3318
3319         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3320                                           GFP_KERNEL, &paddr);
3321         if (!io_task->cmd_bhs)
3322                 return -ENOMEM;
3323         io_task->bhs_pa.u.a64.address = paddr;
3324         io_task->libiscsi_itt = (itt_t)task->itt;
3325         io_task->pwrb_handle = alloc_wrb_handle(phba,
3326                                                 beiscsi_conn->beiscsi_conn_cid -
3327                                                 phba->fw_config.iscsi_cid_start
3328                                                 );
3329         io_task->conn = beiscsi_conn;
3330
3331         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3332         task->hdr_max = sizeof(struct be_cmd_bhs);
3333
3334         if (task->sc) {
3335                 spin_lock(&phba->io_sgl_lock);
3336                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3337                 spin_unlock(&phba->io_sgl_lock);
3338                 if (!io_task->psgl_handle)
3339                         goto free_hndls;
3340         } else {
3341                 io_task->scsi_cmnd = NULL;
3342                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3343                         if (!beiscsi_conn->login_in_progress) {
3344                                 spin_lock(&phba->mgmt_sgl_lock);
3345                                 io_task->psgl_handle = (struct sgl_handle *)
3346                                                 alloc_mgmt_sgl_handle(phba);
3347                                 spin_unlock(&phba->mgmt_sgl_lock);
3348                                 if (!io_task->psgl_handle)
3349                                         goto free_hndls;
3350
3351                                 beiscsi_conn->login_in_progress = 1;
3352                                 beiscsi_conn->plogin_sgl_handle =
3353                                                         io_task->psgl_handle;
3354                         } else {
3355                                 io_task->psgl_handle =
3356                                                 beiscsi_conn->plogin_sgl_handle;
3357                         }
3358                 } else {
3359                         spin_lock(&phba->mgmt_sgl_lock);
3360                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3361                         spin_unlock(&phba->mgmt_sgl_lock);
3362                         if (!io_task->psgl_handle)
3363                                 goto free_hndls;
3364                 }
3365         }
3366         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3367                                  wrb_index << 16) | (unsigned int)
3368                                 (io_task->psgl_handle->sgl_index));
3369         if (ring_mode) {
3370                 phba->sgl_hndl_array[io_task->psgl_handle->sgl_index -
3371                                      phba->fw_config.iscsi_icd_start] =
3372                                      io_task->psgl_handle;
3373                 io_task->psgl_handle->task = task;
3374                 io_task->psgl_handle->cid = beiscsi_conn->beiscsi_conn_cid  -
3375                                             phba->fw_config.iscsi_cid_start;
3376         } else
3377                 io_task->pwrb_handle->pio_handle = task;
3378
3379         io_task->cmd_bhs->iscsi_hdr.itt = itt;
3380         return 0;
3381
3382 free_hndls:
3383         phwi_ctrlr = phba->phwi_ctrlr;
3384         pwrb_context = &phwi_ctrlr->wrb_context[
3385                         beiscsi_conn->beiscsi_conn_cid -
3386                         phba->fw_config.iscsi_cid_start];
3387         free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3388         io_task->pwrb_handle = NULL;
3389         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3390                       io_task->bhs_pa.u.a64.address);
3391         SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed \n");
3392         return -ENOMEM;
3393 }
3394
3395 static void beiscsi_cleanup_task(struct iscsi_task *task)
3396 {
3397         struct beiscsi_io_task *io_task = task->dd_data;
3398         struct iscsi_conn *conn = task->conn;
3399         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3400         struct beiscsi_hba *phba = beiscsi_conn->phba;
3401         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3402         struct hwi_wrb_context *pwrb_context;
3403         struct hwi_controller *phwi_ctrlr;
3404
3405         phwi_ctrlr = phba->phwi_ctrlr;
3406         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3407                         - phba->fw_config.iscsi_cid_start];
3408         if (io_task->pwrb_handle) {
3409                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3410                 io_task->pwrb_handle = NULL;
3411         }
3412
3413         if (io_task->cmd_bhs) {
3414                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3415                               io_task->bhs_pa.u.a64.address);
3416         }
3417
3418         if (task->sc) {
3419                 if (io_task->psgl_handle) {
3420                         spin_lock(&phba->io_sgl_lock);
3421                         free_io_sgl_handle(phba, io_task->psgl_handle);
3422                         spin_unlock(&phba->io_sgl_lock);
3423                         io_task->psgl_handle = NULL;
3424                 }
3425         } else {
3426                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
3427                         return;
3428                 if (io_task->psgl_handle) {
3429                         spin_lock(&phba->mgmt_sgl_lock);
3430                         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3431                         spin_unlock(&phba->mgmt_sgl_lock);
3432                         io_task->psgl_handle = NULL;
3433                 }
3434         }
3435 }
3436
3437 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3438                           unsigned int num_sg, unsigned int xferlen,
3439                           unsigned int writedir)
3440 {
3441
3442         struct beiscsi_io_task *io_task = task->dd_data;
3443         struct iscsi_conn *conn = task->conn;
3444         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3445         struct beiscsi_hba *phba = beiscsi_conn->phba;
3446         struct iscsi_wrb *pwrb = NULL;
3447         unsigned int doorbell = 0;
3448
3449         pwrb = io_task->pwrb_handle->pwrb;
3450         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3451         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3452
3453         if (writedir) {
3454                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3455                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3456                               &io_task->cmd_bhs->iscsi_data_pdu,
3457                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3458                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3459                               &io_task->cmd_bhs->iscsi_data_pdu,
3460                               ISCSI_OPCODE_SCSI_DATA_OUT);
3461                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3462                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3463                 if (ring_mode)
3464                         io_task->psgl_handle->type = INI_WR_CMD;
3465                 else
3466                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3467                                       INI_WR_CMD);
3468                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3469         } else {
3470                 if (ring_mode)
3471                         io_task->psgl_handle->type = INI_RD_CMD;
3472                 else
3473                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3474                                       INI_RD_CMD);
3475                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3476         }
3477         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3478                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3479                io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3480
3481         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3482                       cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3483                                   lun[0]));
3484         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3485         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3486                       io_task->pwrb_handle->wrb_index);
3487         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3488                       be32_to_cpu(task->cmdsn));
3489         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3490                       io_task->psgl_handle->sgl_index);
3491
3492         hwi_write_sgl(pwrb, sg, num_sg, io_task);
3493
3494         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3495                       io_task->pwrb_handle->nxt_wrb_index);
3496         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3497
3498         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3499         if (!ring_mode)
3500                 doorbell |= (io_task->pwrb_handle->wrb_index &
3501                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3502         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3503
3504         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3505         return 0;
3506 }
3507
3508 static int beiscsi_mtask(struct iscsi_task *task)
3509 {
3510         struct beiscsi_io_task *aborted_io_task, *io_task = task->dd_data;
3511         struct iscsi_conn *conn = task->conn;
3512         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3513         struct beiscsi_hba *phba = beiscsi_conn->phba;
3514         struct iscsi_session *session;
3515         struct iscsi_wrb *pwrb = NULL;
3516         struct hwi_controller *phwi_ctrlr;
3517         struct hwi_wrb_context *pwrb_context;
3518         struct wrb_handle *pwrb_handle;
3519         unsigned int doorbell = 0;
3520         unsigned int i, cid;
3521         struct iscsi_task *aborted_task;
3522
3523         cid = beiscsi_conn->beiscsi_conn_cid;
3524         pwrb = io_task->pwrb_handle->pwrb;
3525         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3526                       be32_to_cpu(task->cmdsn));
3527         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3528                       io_task->pwrb_handle->wrb_index);
3529         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3530                       io_task->psgl_handle->sgl_index);
3531
3532         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3533         case ISCSI_OP_LOGIN:
3534                 if (ring_mode)
3535                         io_task->psgl_handle->type = TGT_DM_CMD;
3536                 else
3537                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3538                                       TGT_DM_CMD);
3539                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3540                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3541                 hwi_write_buffer(pwrb, task);
3542                 break;
3543         case ISCSI_OP_NOOP_OUT:
3544                 if (ring_mode)
3545                         io_task->psgl_handle->type = INI_RD_CMD;
3546                 else
3547                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3548                                       INI_RD_CMD);
3549                 hwi_write_buffer(pwrb, task);
3550                 break;
3551         case ISCSI_OP_TEXT:
3552                 if (ring_mode)
3553                         io_task->psgl_handle->type = INI_WR_CMD;
3554                 else
3555                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3556                                       INI_WR_CMD);
3557                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3558                 hwi_write_buffer(pwrb, task);
3559                 break;
3560         case ISCSI_OP_SCSI_TMFUNC:
3561                 session = conn->session;
3562                 i = ((struct iscsi_tm *)task->hdr)->rtt;
3563                 phwi_ctrlr = phba->phwi_ctrlr;
3564                 pwrb_context = &phwi_ctrlr->wrb_context[cid -
3565                                             phba->fw_config.iscsi_cid_start];
3566                 pwrb_handle = pwrb_context->pwrb_handle_basestd[be32_to_cpu(i)
3567                                                                 >> 16];
3568                 aborted_task = pwrb_handle->pio_handle;
3569                  if (!aborted_task)
3570                         return 0;
3571
3572                 aborted_io_task = aborted_task->dd_data;
3573                 if (!aborted_io_task->scsi_cmnd)
3574                         return 0;
3575
3576                 mgmt_invalidate_icds(phba,
3577                                      aborted_io_task->psgl_handle->sgl_index,
3578                                      cid);
3579                 if (ring_mode)
3580                         io_task->psgl_handle->type = INI_TMF_CMD;
3581                 else
3582                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3583                                       INI_TMF_CMD);
3584                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3585                 hwi_write_buffer(pwrb, task);
3586                 break;
3587         case ISCSI_OP_LOGOUT:
3588                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3589                 if (ring_mode)
3590                         io_task->psgl_handle->type = HWH_TYPE_LOGOUT;
3591                 else
3592                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3593                                 HWH_TYPE_LOGOUT);
3594                 hwi_write_buffer(pwrb, task);
3595                 break;
3596
3597         default:
3598                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3599                          task->hdr->opcode & ISCSI_OPCODE_MASK);
3600                 return -EINVAL;
3601         }
3602
3603         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3604                       be32_to_cpu(task->data_count));
3605         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3606                       io_task->pwrb_handle->nxt_wrb_index);
3607         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3608
3609         doorbell |= cid & DB_WRB_POST_CID_MASK;
3610         if (!ring_mode)
3611                 doorbell |= (io_task->pwrb_handle->wrb_index &
3612                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3613         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3614         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3615         return 0;
3616 }
3617
3618 static int beiscsi_task_xmit(struct iscsi_task *task)
3619 {
3620         struct iscsi_conn *conn = task->conn;
3621         struct beiscsi_io_task *io_task = task->dd_data;
3622         struct scsi_cmnd *sc = task->sc;
3623         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3624         struct scatterlist *sg;
3625         int num_sg;
3626         unsigned int  writedir = 0, xferlen = 0;
3627
3628         SE_DEBUG(DBG_LVL_4, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t"
3629                  "beiscsi_conn=%p \n", beiscsi_conn->beiscsi_conn_cid,
3630                  task, conn, beiscsi_conn);
3631         if (!sc)
3632                 return beiscsi_mtask(task);
3633
3634         io_task->scsi_cmnd = sc;
3635         num_sg = scsi_dma_map(sc);
3636         if (num_sg < 0) {
3637                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3638                 return num_sg;
3639         }
3640         SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3641                   (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3642         xferlen = scsi_bufflen(sc);
3643         sg = scsi_sglist(sc);
3644         if (sc->sc_data_direction == DMA_TO_DEVICE) {
3645                 writedir = 1;
3646                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3647                          task->imm_count);
3648         } else
3649                 writedir = 0;
3650         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3651 }
3652
3653
3654 static void beiscsi_remove(struct pci_dev *pcidev)
3655 {
3656         struct beiscsi_hba *phba = NULL;
3657         struct hwi_controller *phwi_ctrlr;
3658         struct hwi_context_memory *phwi_context;
3659         struct be_eq_obj *pbe_eq;
3660         unsigned int i, msix_vec;
3661
3662         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3663         if (!phba) {
3664                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3665                 return;
3666         }
3667
3668         phwi_ctrlr = phba->phwi_ctrlr;
3669         phwi_context = phwi_ctrlr->phwi_ctxt;
3670         hwi_disable_intr(phba);
3671         if (phba->msix_enabled) {
3672                 for (i = 0; i <= phba->num_cpus; i++) {
3673                         msix_vec = phba->msix_entries[i].vector;
3674                         free_irq(msix_vec, &phwi_context->be_eq[i]);
3675                 }
3676         } else
3677                 if (phba->pcidev->irq)
3678                         free_irq(phba->pcidev->irq, phba);
3679         pci_disable_msix(phba->pcidev);
3680         destroy_workqueue(phba->wq);
3681         if (blk_iopoll_enabled)
3682                 for (i = 0; i < phba->num_cpus; i++) {
3683                         pbe_eq = &phwi_context->be_eq[i];
3684                         blk_iopoll_disable(&pbe_eq->iopoll);
3685                 }
3686
3687         beiscsi_clean_port(phba);
3688         beiscsi_free_mem(phba);
3689         beiscsi_unmap_pci_function(phba);
3690         pci_free_consistent(phba->pcidev,
3691                             phba->ctrl.mbox_mem_alloced.size,
3692                             phba->ctrl.mbox_mem_alloced.va,
3693                             phba->ctrl.mbox_mem_alloced.dma);
3694         iscsi_host_remove(phba->shost);
3695         pci_dev_put(phba->pcidev);
3696         iscsi_host_free(phba->shost);
3697 }
3698
3699 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
3700 {
3701         int i, status;
3702
3703         for (i = 0; i <= phba->num_cpus; i++)
3704                 phba->msix_entries[i].entry = i;
3705
3706         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
3707                                  (phba->num_cpus + 1));
3708         if (!status)
3709                 phba->msix_enabled = true;
3710
3711         return;
3712 }
3713
3714 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3715                                 const struct pci_device_id *id)
3716 {
3717         struct beiscsi_hba *phba = NULL;
3718         struct hwi_controller *phwi_ctrlr;
3719         struct hwi_context_memory *phwi_context;
3720         struct be_eq_obj *pbe_eq;
3721         int ret, msix_vec, num_cpus, i;
3722
3723         ret = beiscsi_enable_pci(pcidev);
3724         if (ret < 0) {
3725                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3726                              "Failed to enable pci device \n");
3727                 return ret;
3728         }
3729
3730         phba = beiscsi_hba_alloc(pcidev);
3731         if (!phba) {
3732                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3733                         " Failed in beiscsi_hba_alloc \n");
3734                 goto disable_pci;
3735         }
3736         SE_DEBUG(DBG_LVL_8, " phba = %p \n", phba);
3737
3738         if (enable_msix)
3739                 num_cpus = find_num_cpus();
3740         else
3741                 num_cpus = 1;
3742         phba->num_cpus = num_cpus;
3743         SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", phba->num_cpus);
3744
3745         if (enable_msix)
3746                 beiscsi_msix_enable(phba);
3747         ret = be_ctrl_init(phba, pcidev);
3748         if (ret) {
3749                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3750                                 "Failed in be_ctrl_init\n");
3751                 goto hba_free;
3752         }
3753
3754         spin_lock_init(&phba->io_sgl_lock);
3755         spin_lock_init(&phba->mgmt_sgl_lock);
3756         spin_lock_init(&phba->isr_lock);
3757         ret = mgmt_get_fw_config(&phba->ctrl, phba);
3758         if (ret != 0) {
3759                 shost_printk(KERN_ERR, phba->shost,
3760                              "Error getting fw config\n");
3761                 goto free_port;
3762         }
3763         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
3764         phba->shost->can_queue = phba->params.ios_per_ctrl;
3765         beiscsi_get_params(phba);
3766         ret = beiscsi_init_port(phba);
3767         if (ret < 0) {
3768                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3769                              "Failed in beiscsi_init_port\n");
3770                 goto free_port;
3771         }
3772
3773         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3774                  phba->shost->host_no);
3775         phba->wq = create_workqueue(phba->wq_name);
3776         if (!phba->wq) {
3777                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3778                                 "Failed to allocate work queue\n");
3779                 goto free_twq;
3780         }
3781
3782         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3783
3784         phwi_ctrlr = phba->phwi_ctrlr;
3785         phwi_context = phwi_ctrlr->phwi_ctxt;
3786         if (blk_iopoll_enabled) {
3787                 for (i = 0; i < phba->num_cpus; i++) {
3788                         pbe_eq = &phwi_context->be_eq[i];
3789                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
3790                                         be_iopoll);
3791                         blk_iopoll_enable(&pbe_eq->iopoll);
3792                 }
3793         }
3794         ret = beiscsi_init_irqs(phba);
3795         if (ret < 0) {
3796                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3797                              "Failed to beiscsi_init_irqs\n");
3798                 goto free_blkenbld;
3799         }
3800         ret = hwi_enable_intr(phba);
3801         if (ret < 0) {
3802                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3803                              "Failed to hwi_enable_intr\n");
3804                 goto free_ctrlr;
3805         }
3806         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3807         return 0;
3808
3809 free_ctrlr:
3810         if (phba->msix_enabled) {
3811                 for (i = 0; i <= phba->num_cpus; i++) {
3812                         msix_vec = phba->msix_entries[i].vector;
3813                         free_irq(msix_vec, &phwi_context->be_eq[i]);
3814                 }
3815         } else
3816                 if (phba->pcidev->irq)
3817                         free_irq(phba->pcidev->irq, phba);
3818         pci_disable_msix(phba->pcidev);
3819 free_blkenbld:
3820         destroy_workqueue(phba->wq);
3821         if (blk_iopoll_enabled)
3822                 for (i = 0; i < phba->num_cpus; i++) {
3823                         pbe_eq = &phwi_context->be_eq[i];
3824                         blk_iopoll_disable(&pbe_eq->iopoll);
3825                 }
3826 free_twq:
3827         beiscsi_clean_port(phba);
3828         beiscsi_free_mem(phba);
3829 free_port:
3830         pci_free_consistent(phba->pcidev,
3831                             phba->ctrl.mbox_mem_alloced.size,
3832                             phba->ctrl.mbox_mem_alloced.va,
3833                            phba->ctrl.mbox_mem_alloced.dma);
3834         beiscsi_unmap_pci_function(phba);
3835 hba_free:
3836         iscsi_host_remove(phba->shost);
3837         pci_dev_put(phba->pcidev);
3838         iscsi_host_free(phba->shost);
3839 disable_pci:
3840         pci_disable_device(pcidev);
3841         return ret;
3842 }
3843
3844 struct iscsi_transport beiscsi_iscsi_transport = {
3845         .owner = THIS_MODULE,
3846         .name = DRV_NAME,
3847         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST |
3848                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3849         .param_mask = ISCSI_MAX_RECV_DLENGTH |
3850                 ISCSI_MAX_XMIT_DLENGTH |
3851                 ISCSI_HDRDGST_EN |
3852                 ISCSI_DATADGST_EN |
3853                 ISCSI_INITIAL_R2T_EN |
3854                 ISCSI_MAX_R2T |
3855                 ISCSI_IMM_DATA_EN |
3856                 ISCSI_FIRST_BURST |
3857                 ISCSI_MAX_BURST |
3858                 ISCSI_PDU_INORDER_EN |
3859                 ISCSI_DATASEQ_INORDER_EN |
3860                 ISCSI_ERL |
3861                 ISCSI_CONN_PORT |
3862                 ISCSI_CONN_ADDRESS |
3863                 ISCSI_EXP_STATSN |
3864                 ISCSI_PERSISTENT_PORT |
3865                 ISCSI_PERSISTENT_ADDRESS |
3866                 ISCSI_TARGET_NAME | ISCSI_TPGT |
3867                 ISCSI_USERNAME | ISCSI_PASSWORD |
3868                 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3869                 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3870                 ISCSI_LU_RESET_TMO |
3871                 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3872                 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3873         .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3874                                 ISCSI_HOST_INITIATOR_NAME,
3875         .create_session = beiscsi_session_create,
3876         .destroy_session = beiscsi_session_destroy,
3877         .create_conn = beiscsi_conn_create,
3878         .bind_conn = beiscsi_conn_bind,
3879         .destroy_conn = iscsi_conn_teardown,
3880         .set_param = beiscsi_set_param,
3881         .get_conn_param = beiscsi_conn_get_param,
3882         .get_session_param = iscsi_session_get_param,
3883         .get_host_param = beiscsi_get_host_param,
3884         .start_conn = beiscsi_conn_start,
3885         .stop_conn = beiscsi_conn_stop,
3886         .send_pdu = iscsi_conn_send_pdu,
3887         .xmit_task = beiscsi_task_xmit,
3888         .cleanup_task = beiscsi_cleanup_task,
3889         .alloc_pdu = beiscsi_alloc_pdu,
3890         .parse_pdu_itt = beiscsi_parse_pdu,
3891         .get_stats = beiscsi_conn_get_stats,
3892         .ep_connect = beiscsi_ep_connect,
3893         .ep_poll = beiscsi_ep_poll,
3894         .ep_disconnect = beiscsi_ep_disconnect,
3895         .session_recovery_timedout = iscsi_session_recovery_timedout,
3896 };
3897
3898 static struct pci_driver beiscsi_pci_driver = {
3899         .name = DRV_NAME,
3900         .probe = beiscsi_dev_probe,
3901         .remove = beiscsi_remove,
3902         .id_table = beiscsi_pci_id_table
3903 };
3904
3905
3906 static int __init beiscsi_module_init(void)
3907 {
3908         int ret;
3909
3910         beiscsi_scsi_transport =
3911                         iscsi_register_transport(&beiscsi_iscsi_transport);
3912         if (!beiscsi_scsi_transport) {
3913                 SE_DEBUG(DBG_LVL_1,
3914                          "beiscsi_module_init - Unable to  register beiscsi"
3915                          "transport.\n");
3916                 ret = -ENOMEM;
3917         }
3918         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3919                  &beiscsi_iscsi_transport);
3920
3921         ret = pci_register_driver(&beiscsi_pci_driver);
3922         if (ret) {
3923                 SE_DEBUG(DBG_LVL_1,
3924                          "beiscsi_module_init - Unable to  register"
3925                          "beiscsi pci driver.\n");
3926                 goto unregister_iscsi_transport;
3927         }
3928         ring_mode = 0;
3929         return 0;
3930
3931 unregister_iscsi_transport:
3932         iscsi_unregister_transport(&beiscsi_iscsi_transport);
3933         return ret;
3934 }
3935
3936 static void __exit beiscsi_module_exit(void)
3937 {
3938         pci_unregister_driver(&beiscsi_pci_driver);
3939         iscsi_unregister_transport(&beiscsi_iscsi_transport);
3940 }
3941
3942 module_init(beiscsi_module_init);
3943 module_exit(beiscsi_module_exit);