SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel-NE, Inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77
  78 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  79 static unsigned char *nes_iwarp_state_str[] = {
  80         "Non-Existant",
  81         "Idle",
  82         "RTS",
  83         "Closing",
  84         "RSVD1",
  85         "Terminate",
  86         "Error",
  87         "RSVD2",
  88 };
  89
  90 static unsigned char *nes_tcp_state_str[] = {
  91         "Non-Existant",
  92         "Closed",
  93         "Listen",
  94         "SYN Sent",
  95         "SYN Rcvd",
  96         "Established",
  97         "Close Wait",
  98         "FIN Wait 1",
  99         "Closing",
 100         "Last Ack",
 101         "FIN Wait 2",
 102         "Time Wait",
 103         "RSVD1",
 104         "RSVD2",
 105         "RSVD3",
 106         "RSVD4",
 107 };
 108 #endif
 109
 110
 111 /**
 112  * nes_nic_init_timer_defaults
 113  */
 114 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 115 {
 116         unsigned long flags;
 117         struct nes_adapter *nesadapter = nesdev->nesadapter;
 118         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 119
 120         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 121
 122         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 123         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 124         if (jumbomode) {
 125                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 126                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 127                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 128         } else {
 129                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 130                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 131                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 132         }
 133
 134         /* todo use netdev->mtu to set thresholds */
 135         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 136 }
 137
 138
 139 /**
 140  * nes_nic_init_timer
 141  */
 142 static void  nes_nic_init_timer(struct nes_device *nesdev)
 143 {
 144         unsigned long flags;
 145         struct nes_adapter *nesadapter = nesdev->nesadapter;
 146         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 147
 148         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 149
 150         if (shared_timer->timer_in_use_old == 0) {
 151                 nesdev->deepcq_count = 0;
 152                 shared_timer->timer_direction_upward = 0;
 153                 shared_timer->timer_direction_downward = 0;
 154                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 155                 shared_timer->timer_in_use_old = 0;
 156
 157         }
 158         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 159                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 160                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 161                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 162         }
 163         /* todo use netdev->mtu to set thresholds */
 164         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 165 }
 166
 167
 168 /**
 169  * nes_nic_tune_timer
 170  */
 171 static void nes_nic_tune_timer(struct nes_device *nesdev)
 172 {
 173         unsigned long flags;
 174         struct nes_adapter *nesadapter = nesdev->nesadapter;
 175         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 176         u16 cq_count = nesdev->currcq_count;
 177
 178         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 179
 180         if (shared_timer->cq_count_old <= cq_count)
 181                 shared_timer->cq_direction_downward = 0;
 182         else
 183                 shared_timer->cq_direction_downward++;
 184         shared_timer->cq_count_old = cq_count;
 185         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 186                 if (cq_count <= shared_timer->threshold_low &&
 187                     shared_timer->threshold_low > 4) {
 188                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 189                         shared_timer->cq_direction_downward=0;
 190                         nesdev->currcq_count = 0;
 191                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 192                         return;
 193                 }
 194         }
 195
 196         if (cq_count > 1) {
 197                 nesdev->deepcq_count += cq_count;
 198                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 199                         shared_timer->timer_direction_upward++;
 200                         shared_timer->timer_direction_downward = 0;
 201                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 202                         shared_timer->timer_direction_upward = 0;
 203                         shared_timer->timer_direction_downward = 0;
 204                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 205                         shared_timer->timer_direction_downward++;
 206                         shared_timer->timer_direction_upward = 0;
 207                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 208                         shared_timer->timer_in_use -= 2;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward++;
 211                 } else {
 212                         shared_timer->timer_in_use -= 4;
 213                         shared_timer->timer_direction_upward = 0;
 214                         shared_timer->timer_direction_downward++;
 215                 }
 216
 217                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 218                         shared_timer->timer_in_use += 3;
 219                         shared_timer->timer_direction_upward = 0;
 220                         shared_timer->timer_direction_downward = 0;
 221                 }
 222                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 223                         shared_timer->timer_in_use -= 4 ;
 224                         shared_timer->timer_direction_downward = 0;
 225                         shared_timer->timer_direction_upward = 0;
 226                 }
 227         }
 228
 229         /* boundary checking */
 230         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 231                 shared_timer->timer_in_use = shared_timer->threshold_high;
 232         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 233                 shared_timer->timer_in_use = shared_timer->threshold_low;
 234
 235         nesdev->currcq_count = 0;
 236
 237         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 238 }
 239
 240
 241 /**
 242  * nes_init_adapter - initialize adapter
 243  */
 244 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 245         struct nes_adapter *nesadapter = NULL;
 246         unsigned long num_pds;
 247         u32 u32temp;
 248         u32 port_count;
 249         u16 max_rq_wrs;
 250         u16 max_sq_wrs;
 251         u32 max_mr;
 252         u32 max_256pbl;
 253         u32 max_4kpbl;
 254         u32 max_qp;
 255         u32 max_irrq;
 256         u32 max_cq;
 257         u32 hte_index_mask;
 258         u32 adapter_size;
 259         u32 arp_table_size;
 260         u16 vendor_id;
 261         u16 device_id;
 262         u8  OneG_Mode;
 263         u8  func_index;
 264
 265         /* search the list of existing adapters */
 266         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 267                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 268                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 269                                 nesdev->pcidev->devfn,
 270                                 PCI_SLOT(nesadapter->devfn),
 271                                 nesadapter->bus_number,
 272                                 PCI_SLOT(nesdev->pcidev->devfn),
 273                                 nesdev->pcidev->bus->number );
 274                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 275                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 276                         nesadapter->ref_count++;
 277                         return nesadapter;
 278                 }
 279         }
 280
 281         /* no adapter found */
 282         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 283         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 284                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 285                                 hw_rev);
 286                 return NULL;
 287         }
 288
 289         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 290                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 291                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 292                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 294
 295         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 296
 297
 298         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 299                 return NULL;
 300
 301         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 302         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 303
 304         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 305         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 306                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 307                                 max_qp, u32temp);
 308                 max_qp = (u32)1 << (u32temp & 0x001f);
 309         }
 310
 311         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 312         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 313                         max_qp, hte_index_mask);
 314
 315         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 316
 317         max_irrq = 1 << (u32temp & 0x001f);
 318
 319         if (max_qp > max_irrq) {
 320                 max_qp = max_irrq;
 321                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 322                                 max_qp);
 323         }
 324
 325         /* there should be no reason to allocate more pds than qps */
 326         if (num_pds > max_qp)
 327                 num_pds = max_qp;
 328
 329         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 330         max_mr = (u32)8192 << (u32temp & 0x7);
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 333         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 334         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 335         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 336
 337         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 338         arp_table_size = 1 << u32temp;
 339
 340         adapter_size = (sizeof(struct nes_adapter) +
 341                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 342         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 343         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 347         adapter_size += sizeof(struct nes_qp **) * max_qp;
 348
 349         /* allocate a new adapter struct */
 350         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 351         if (nesadapter == NULL) {
 352                 return NULL;
 353         }
 354
 355         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 356                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 357
 358         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 359                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 360                 kfree(nesadapter);
 361                 return NULL;
 362         }
 363
 364         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 365                                 (nesadapter->mac_addr_low >> 24);
 366
 367         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 368                                  PCI_DEVICE_ID, &device_id);
 369         nesadapter->vendor_part_id = device_id;
 370
 371         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 372                                                         OneG_Mode)) {
 373                 kfree(nesadapter);
 374                 return NULL;
 375         }
 376         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 377
 378         memset(nesadapter->pft_mcast_map, 255,
 379                sizeof nesadapter->pft_mcast_map);
 380
 381         /* populate the new nesadapter */
 382         nesadapter->devfn = nesdev->pcidev->devfn;
 383         nesadapter->bus_number = nesdev->pcidev->bus->number;
 384         nesadapter->ref_count = 1;
 385         nesadapter->timer_int_req = 0xffff0000;
 386         nesadapter->OneG_Mode = OneG_Mode;
 387         nesadapter->doorbell_start = nesdev->doorbell_region;
 388
 389         /* nesadapter->tick_delta = clk_divisor; */
 390         nesadapter->hw_rev = hw_rev;
 391         nesadapter->port_count = port_count;
 392
 393         nesadapter->max_qp = max_qp;
 394         nesadapter->hte_index_mask = hte_index_mask;
 395         nesadapter->max_irrq = max_irrq;
 396         nesadapter->max_mr = max_mr;
 397         nesadapter->max_256pbl = max_256pbl - 1;
 398         nesadapter->max_4kpbl = max_4kpbl - 1;
 399         nesadapter->max_cq = max_cq;
 400         nesadapter->free_256pbl = max_256pbl - 1;
 401         nesadapter->free_4kpbl = max_4kpbl - 1;
 402         nesadapter->max_pd = num_pds;
 403         nesadapter->arp_table_size = arp_table_size;
 404
 405         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 406         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 407                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 408                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 409                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 410         } else {
 411                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 412                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 413                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 414                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 415         }
 416         /* Setup and enable the periodic timer */
 417         if (nesadapter->et_rx_coalesce_usecs_irq)
 418                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 419                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 420         else
 421                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 422
 423         nesadapter->base_pd = 1;
 424
 425         nesadapter->device_cap_flags =
 426                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 427
 428         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 429                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 430         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 431         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 432         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 433         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 434         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 435
 436
 437         /* mark the usual suspect QPs and CQs as in use */
 438         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 439                 set_bit(u32temp, nesadapter->allocated_qps);
 440                 set_bit(u32temp, nesadapter->allocated_cqs);
 441         }
 442
 443         for (u32temp = 0; u32temp < 20; u32temp++)
 444                 set_bit(u32temp, nesadapter->allocated_pds);
 445         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 446
 447         max_rq_wrs = ((u32temp >> 8) & 3);
 448         switch (max_rq_wrs) {
 449                 case 0:
 450                         max_rq_wrs = 4;
 451                         break;
 452                 case 1:
 453                         max_rq_wrs = 16;
 454                         break;
 455                 case 2:
 456                         max_rq_wrs = 32;
 457                         break;
 458                 case 3:
 459                         max_rq_wrs = 512;
 460                         break;
 461         }
 462
 463         max_sq_wrs = (u32temp & 3);
 464         switch (max_sq_wrs) {
 465                 case 0:
 466                         max_sq_wrs = 4;
 467                         break;
 468                 case 1:
 469                         max_sq_wrs = 16;
 470                         break;
 471                 case 2:
 472                         max_sq_wrs = 32;
 473                         break;
 474                 case 3:
 475                         max_sq_wrs = 512;
 476                         break;
 477         }
 478         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 479         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 480
 481         nesadapter->max_sge = 4;
 482         nesadapter->max_cqe = 32767;
 483
 484         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 485                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 486                 kfree(nesadapter);
 487                 return NULL;
 488         }
 489
 490         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 491         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 492                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 493
 494         /* setup port configuration */
 495         if (nesadapter->port_count == 1) {
 496                 nesadapter->log_port = 0x00000000;
 497                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 498                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 499                 else
 500                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 501         } else {
 502                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 503                         nesadapter->log_port = 0x000000D8;
 504                 } else {
 505                         if (nesadapter->port_count == 2)
 506                                 nesadapter->log_port = 0x00000044;
 507                         else
 508                                 nesadapter->log_port = 0x000000e4;
 509                 }
 510                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 511         }
 512
 513         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 514                                                 nesadapter->log_port);
 515         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 516                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 517
 518         spin_lock_init(&nesadapter->resource_lock);
 519         spin_lock_init(&nesadapter->phy_lock);
 520         spin_lock_init(&nesadapter->pbl_lock);
 521         spin_lock_init(&nesadapter->periodic_timer_lock);
 522
 523         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 524         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 525         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 526         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 527
 528         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 529                 u32 pcs_control_status0, pcs_control_status1;
 530                 u32 reset_value;
 531                 u32 i = 0;
 532                 u32 int_cnt = 0;
 533                 u32 ext_cnt = 0;
 534                 unsigned long flags;
 535                 u32 j = 0;
 536
 537                 pcs_control_status0 = nes_read_indexed(nesdev,
 538                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 539                 pcs_control_status1 = nes_read_indexed(nesdev,
 540                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 541
 542                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 543                         pcs_control_status0 = nes_read_indexed(nesdev,
 544                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 545                         pcs_control_status1 = nes_read_indexed(nesdev,
 546                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 547                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 548                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 549                                 int_cnt++;
 550                         msleep(1);
 551                 }
 552                 if (int_cnt > 1) {
 553                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 554                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 555                         mh_detected++;
 556                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 557                         reset_value |= 0x0000003d;
 558                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 559
 560                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 561                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 562                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 563
 564                         pcs_control_status0 = nes_read_indexed(nesdev,
 565                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 566                         pcs_control_status1 = nes_read_indexed(nesdev,
 567                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 568
 569                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 570                                 pcs_control_status0 = nes_read_indexed(nesdev,
 571                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 572                                 pcs_control_status1 = nes_read_indexed(nesdev,
 573                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 574                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 575                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 576                                         if (++ext_cnt > int_cnt) {
 577                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 578                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 579                                                                 0x0000F088);
 580                                                 mh_detected++;
 581                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 582                                                 reset_value |= 0x0000003d;
 583                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 584
 585                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 586                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 587                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 588                                                 break;
 589                                         }
 590                                 }
 591                                 msleep(1);
 592                         }
 593                 }
 594         }
 595
 596         if (nesadapter->hw_rev == NE020_REV) {
 597                 init_timer(&nesadapter->mh_timer);
 598                 nesadapter->mh_timer.function = nes_mh_fix;
 599                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 600                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 601                 add_timer(&nesadapter->mh_timer);
 602         } else {
 603                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 604         }
 605
 606         init_timer(&nesadapter->lc_timer);
 607         nesadapter->lc_timer.function = nes_clc;
 608         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 609         nesadapter->lc_timer.data = (unsigned long)nesdev;
 610         add_timer(&nesadapter->lc_timer);
 611
 612         list_add_tail(&nesadapter->list, &nes_adapter_list);
 613
 614         for (func_index = 0; func_index < 8; func_index++) {
 615                 pci_bus_read_config_word(nesdev->pcidev->bus,
 616                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 617                                         func_index), 0, &vendor_id);
 618                 if (vendor_id == 0xffff)
 619                         break;
 620         }
 621         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 622                 func_index, pci_name(nesdev->pcidev));
 623         nesadapter->adapter_fcn_count = func_index;
 624
 625         return nesadapter;
 626 }
 627
 628
 629 /**
 630  * nes_reset_adapter_ne020
 631  */
 632 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 633 {
 634         u32 port_count;
 635         u32 u32temp;
 636         u32 i;
 637
 638         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 639         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 640         /* TODO: assuming that both SERDES are set the same for now */
 641         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 642         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 643                         u32temp, port_count);
 644         if (*OneG_Mode)
 645                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 646         u32temp &= 0xff00ffc0;
 647         switch (port_count) {
 648                 case 1:
 649                         u32temp |= 0x00ee0000;
 650                         break;
 651                 case 2:
 652                         u32temp |= 0x00cc0000;
 653                         break;
 654                 case 4:
 655                         u32temp |= 0x00000000;
 656                         break;
 657                 default:
 658                         return 0;
 659                         break;
 660         }
 661
 662         /* check and do full reset if needed */
 663         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 664                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 665                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 666
 667                 i = 0;
 668                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 669                         mdelay(1);
 670                 if (i >= 10000) {
 671                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 672                         return 0;
 673                 }
 674
 675                 i = 0;
 676                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 677                         mdelay(1);
 678                 if (i >= 10000) {
 679                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 680                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 681                         return 0;
 682                 }
 683         }
 684
 685         /* port reset */
 686         switch (port_count) {
 687                 case 1:
 688                         u32temp |= 0x00ee0010;
 689                         break;
 690                 case 2:
 691                         u32temp |= 0x00cc0030;
 692                         break;
 693                 case 4:
 694                         u32temp |= 0x00000030;
 695                         break;
 696         }
 697
 698         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 699         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 700
 701         i = 0;
 702         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 703                 mdelay(1);
 704         if (i >= 10000) {
 705                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 706                 return 0;
 707         }
 708
 709         /* serdes 0 */
 710         i = 0;
 711         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 712                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 713                 mdelay(1);
 714         if (i >= 5000) {
 715                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 716                 return 0;
 717         }
 718
 719         /* serdes 1 */
 720         if (port_count > 1) {
 721                 i = 0;
 722                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 723                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 724                         mdelay(1);
 725                 if (i >= 5000) {
 726                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 727                         return 0;
 728                 }
 729         }
 730
 731         return port_count;
 732 }
 733
 734
 735 /**
 736  * nes_init_serdes
 737  */
 738 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 739                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 740 {
 741         int i;
 742         u32 u32temp;
 743         u32 sds;
 744
 745         if (hw_rev != NE020_REV) {
 746                 /* init serdes 0 */
 747                 if (wide_ppm_offset && (nesadapter->phy_type[0] == NES_PHY_TYPE_CX4))
 748                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 749                 else
 750                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 751
 752                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 753                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 754                         sds |= 0x00000100;
 755                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 756                 }
 757                 if (!OneG_Mode)
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 759
 760                 if (port_count < 2)
 761                         return 0;
 762
 763                 /* init serdes 1 */
 764                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 765                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 766
 767                 switch (nesadapter->phy_type[1]) {
 768                 case NES_PHY_TYPE_ARGUS:
 769                 case NES_PHY_TYPE_SFP_D:
 770                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 771                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 772                         break;
 773                 case NES_PHY_TYPE_CX4:
 774                         if (wide_ppm_offset)
 775                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 776                         break;
 777                 case NES_PHY_TYPE_PUMA_1G:
 778                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 779                         sds |= 0x000000100;
 780                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 781                 }
 782                 if (!OneG_Mode) {
 783                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 784                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 785                         sds &= 0xFFFFFFBF;
 786                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 787                 }
 788         } else {
 789                 /* init serdes 0 */
 790                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 791                 i = 0;
 792                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 793                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 794                         mdelay(1);
 795                 if (i >= 5000) {
 796                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 797                         return 1;
 798                 }
 799                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 800                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 801                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 802                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 803                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 805                 if (OneG_Mode)
 806                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 807                 else
 808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 809
 810                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 811                 if (port_count > 1) {
 812                         /* init serdes 1 */
 813                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 814                         i = 0;
 815                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 816                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 817                                 mdelay(1);
 818                         if (i >= 5000) {
 819                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 820                                 /* return 1; */
 821                         }
 822                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 823                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 824                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 829                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 830                 }
 831         }
 832         return 0;
 833 }
 834
 835
 836 /**
 837  * nes_init_csr_ne020
 838  * Initialize registers for ne020 hardware
 839  */
 840 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 841 {
 842         u32 u32temp;
 843
 844         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 845
 846         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 847         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 848         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 849         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 850         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 851         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 852         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 853         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 854
 855         /* TODO: move these MAC register settings to NIC bringup */
 856         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 857         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 858         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 859         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 860         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 861         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 862         if (port_count > 1) {
 863                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 864                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 865                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 866                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 867                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 868                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 869                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 870         }
 871         if (port_count > 2) {
 872                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 873                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 874                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 875                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 876                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 877                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 878                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 879
 880                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 881                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 882                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 883                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 885                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 886                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 887         }
 888
 889         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 890         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 891         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 892                                                          0x00000001);
 893         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 894         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 895         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 896         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 897         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 898
 899         /* TODO: move this to code, get from EEPROM */
 900         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 901         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 902         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 903
 904         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 905         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 906
 907         if (hw_rev != NE020_REV) {
 908                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 909                 u32temp |= 0x80000000;
 910                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 911                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 912                 u32temp &= 0x7fffffff;
 913                 u32temp |= 0x7fff0010;
 914                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 915         }
 916 }
 917
 918
 919 /**
 920  * nes_destroy_adapter - destroy the adapter structure
 921  */
 922 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 923 {
 924         struct nes_adapter *tmp_adapter;
 925
 926         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 927                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 928                                 tmp_adapter);
 929         }
 930
 931         nesadapter->ref_count--;
 932         if (!nesadapter->ref_count) {
 933                 if (nesadapter->hw_rev == NE020_REV) {
 934                         del_timer(&nesadapter->mh_timer);
 935                 }
 936                 del_timer(&nesadapter->lc_timer);
 937
 938                 list_del(&nesadapter->list);
 939                 kfree(nesadapter);
 940         }
 941 }
 942
 943
 944 /**
 945  * nes_init_cqp
 946  */
 947 int nes_init_cqp(struct nes_device *nesdev)
 948 {
 949         struct nes_adapter *nesadapter = nesdev->nesadapter;
 950         struct nes_hw_cqp_qp_context *cqp_qp_context;
 951         struct nes_hw_cqp_wqe *cqp_wqe;
 952         struct nes_hw_ceq *ceq;
 953         struct nes_hw_ceq *nic_ceq;
 954         struct nes_hw_aeq *aeq;
 955         void *vmem;
 956         dma_addr_t pmem;
 957         u32 count=0;
 958         u32 cqp_head;
 959         u64 u64temp;
 960         u32 u32temp;
 961
 962         /* allocate CQP memory */
 963         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 964         /* SQ is 512 byte aligned, others are 256 byte aligned */
 965         nesdev->cqp_mem_size = 512 +
 966                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 967                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 968                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 969                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 970                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 971                         sizeof(struct nes_hw_cqp_qp_context);
 972
 973         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 974                         &nesdev->cqp_pbase);
 975         if (!nesdev->cqp_vbase) {
 976                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 977                 return -ENOMEM;
 978         }
 979         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 980
 981         /* Allocate a twice the number of CQP requests as the SQ size */
 982         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 983                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 984         if (nesdev->nes_cqp_requests == NULL) {
 985                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 986                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 987                                 nesdev->cqp.sq_pbase);
 988                 return -ENOMEM;
 989         }
 990
 991         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 992                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 993
 994         spin_lock_init(&nesdev->cqp.lock);
 995         init_waitqueue_head(&nesdev->cqp.waitq);
 996
 997         /* Setup Various Structures */
 998         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 999                         ~(unsigned long)(512 - 1));
1000         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1001                         ~(unsigned long long)(512 - 1));
1002
1003         nesdev->cqp.sq_vbase = vmem;
1004         nesdev->cqp.sq_pbase = pmem;
1005         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1006         nesdev->cqp.sq_head = 0;
1007         nesdev->cqp.sq_tail = 0;
1008         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1009
1010         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1011         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1012
1013         nesdev->ccq.cq_vbase = vmem;
1014         nesdev->ccq.cq_pbase = pmem;
1015         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1016         nesdev->ccq.cq_head = 0;
1017         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1018         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1019
1020         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1021         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1022
1023         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1024         ceq = &nesadapter->ceq[nesdev->ceq_index];
1025         ceq->ceq_vbase = vmem;
1026         ceq->ceq_pbase = pmem;
1027         ceq->ceq_size = NES_CCEQ_SIZE;
1028         ceq->ceq_head = 0;
1029
1030         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1031         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1032
1033         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1034         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1035         nic_ceq->ceq_vbase = vmem;
1036         nic_ceq->ceq_pbase = pmem;
1037         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1038         nic_ceq->ceq_head = 0;
1039
1040         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1041         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1042
1043         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1044         aeq->aeq_vbase = vmem;
1045         aeq->aeq_pbase = pmem;
1046         aeq->aeq_size = nesadapter->max_qp;
1047         aeq->aeq_head = 0;
1048
1049         /* Setup QP Context */
1050         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1051         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1052
1053         cqp_qp_context = vmem;
1054         cqp_qp_context->context_words[0] =
1055                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1056         cqp_qp_context->context_words[1] = 0;
1057         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1058         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1059
1060
1061         /* Write the address to Create CQP */
1062         if ((sizeof(dma_addr_t) > 4)) {
1063                 nes_write_indexed(nesdev,
1064                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1065                                 ((u64)pmem) >> 32);
1066         } else {
1067                 nes_write_indexed(nesdev,
1068                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1069         }
1070         nes_write_indexed(nesdev,
1071                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1072                         (u32)pmem);
1073
1074         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1075         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1076
1077         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1078                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1079                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1080         }
1081
1082         /* Write Create CCQ WQE */
1083         cqp_head = nesdev->cqp.sq_head++;
1084         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1085         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1086         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1087                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1088                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1089         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1090                             (nesdev->ccq.cq_number |
1091                              ((u32)nesdev->ceq_index << 16)));
1092         u64temp = (u64)nesdev->ccq.cq_pbase;
1093         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1094         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1095         u64temp = (unsigned long)&nesdev->ccq;
1096         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1097                         cpu_to_le32((u32)(u64temp >> 1));
1098         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1099                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1100         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1101
1102         /* Write Create CEQ WQE */
1103         cqp_head = nesdev->cqp.sq_head++;
1104         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1105         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1106         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1107                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1108         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1109         u64temp = (u64)ceq->ceq_pbase;
1110         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1111
1112         /* Write Create AEQ WQE */
1113         cqp_head = nesdev->cqp.sq_head++;
1114         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1115         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1116         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1117                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1118         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1119         u64temp = (u64)aeq->aeq_pbase;
1120         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1121
1122         /* Write Create NIC CEQ WQE */
1123         cqp_head = nesdev->cqp.sq_head++;
1124         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1125         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1126         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1127                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1128         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1129         u64temp = (u64)nic_ceq->ceq_pbase;
1130         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1131
1132         /* Poll until CCQP done */
1133         count = 0;
1134         do {
1135                 if (count++ > 1000) {
1136                         printk(KERN_ERR PFX "Error creating CQP\n");
1137                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1138                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1139                         return -1;
1140                 }
1141                 udelay(10);
1142         } while (!(nes_read_indexed(nesdev,
1143                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1144
1145         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1146                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1147
1148         u32temp = 0x04800000;
1149         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1150
1151         /* wait for the CCQ, CEQ, and AEQ to get created */
1152         count = 0;
1153         do {
1154                 if (count++ > 1000) {
1155                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1156                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1157                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1158                         return -1;
1159                 }
1160                 udelay(10);
1161         } while (((nes_read_indexed(nesdev,
1162                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1163
1164         /* dump the QP status value */
1165         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1166                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1167
1168         nesdev->cqp.sq_tail++;
1169
1170         return 0;
1171 }
1172
1173
1174 /**
1175  * nes_destroy_cqp
1176  */
1177 int nes_destroy_cqp(struct nes_device *nesdev)
1178 {
1179         struct nes_hw_cqp_wqe *cqp_wqe;
1180         u32 count = 0;
1181         u32 cqp_head;
1182         unsigned long flags;
1183
1184         do {
1185                 if (count++ > 1000)
1186                         break;
1187                 udelay(10);
1188         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1189
1190         /* Reset CCQ */
1191         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1192                         nesdev->ccq.cq_number);
1193
1194         /* Disable device interrupts */
1195         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1196
1197         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1198
1199         /* Destroy the AEQ */
1200         cqp_head = nesdev->cqp.sq_head++;
1201         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1202         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1203         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1204                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1205         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1206
1207         /* Destroy the NIC CEQ */
1208         cqp_head = nesdev->cqp.sq_head++;
1209         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1210         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1211         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1212                         ((u32)nesdev->nic_ceq_index << 8));
1213
1214         /* Destroy the CEQ */
1215         cqp_head = nesdev->cqp.sq_head++;
1216         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1217         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1218         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1219                         (nesdev->ceq_index << 8));
1220
1221         /* Destroy the CCQ */
1222         cqp_head = nesdev->cqp.sq_head++;
1223         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1224         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1225         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1226         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1227                         ((u32)nesdev->ceq_index << 16));
1228
1229         /* Destroy CQP */
1230         cqp_head = nesdev->cqp.sq_head++;
1231         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1232         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1233         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1234                         NES_CQP_QP_TYPE_CQP);
1235         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1236
1237         barrier();
1238         /* Ring doorbell (5 WQEs) */
1239         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1240
1241         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1242
1243         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1244         count = 0;
1245         do {
1246                 if (count++ > 1000) {
1247                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1248                                         PCI_FUNC(nesdev->pcidev->devfn));
1249                         break;
1250                 }
1251                 udelay(10);
1252         } while (((nes_read_indexed(nesdev,
1253                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1254
1255         /* dump the QP status value */
1256         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1257                         PCI_FUNC(nesdev->pcidev->devfn),
1258                         nes_read_indexed(nesdev,
1259                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1260
1261         kfree(nesdev->nes_cqp_requests);
1262
1263         /* Free the control structures */
1264         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1265                         nesdev->cqp.sq_pbase);
1266
1267         return 0;
1268 }
1269
1270
1271 /**
1272  * nes_init_phy
1273  */
1274 int nes_init_phy(struct nes_device *nesdev)
1275 {
1276         struct nes_adapter *nesadapter = nesdev->nesadapter;
1277         u32 counter = 0;
1278         u32 sds;
1279         u32 mac_index = nesdev->mac_index;
1280         u32 tx_config = 0;
1281         u16 phy_data;
1282         u32 temp_phy_data = 0;
1283         u32 temp_phy_data2 = 0;
1284         u8  phy_type = nesadapter->phy_type[mac_index];
1285         u8  phy_index = nesadapter->phy_index[mac_index];
1286
1287         if ((nesadapter->OneG_Mode) &&
1288             (phy_type != NES_PHY_TYPE_PUMA_1G)) {
1289                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1290                 if (phy_type == NES_PHY_TYPE_1G) {
1291                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1292                         tx_config &= 0xFFFFFFE3;
1293                         tx_config |= 0x04;
1294                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1295                 }
1296
1297                 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1298                 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1299
1300                 /* Reset the PHY */
1301                 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1302                 udelay(100);
1303                 counter = 0;
1304                 do {
1305                         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1306                         if (counter++ > 100)
1307                                 break;
1308                 } while (phy_data & 0x8000);
1309
1310                 /* Setting no phy loopback */
1311                 phy_data &= 0xbfff;
1312                 phy_data |= 0x1140;
1313                 nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1314                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1315                 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1316                 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1317
1318                 /* Setting the interrupt mask */
1319                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1320                 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1321                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1322
1323                 /* turning on flow control */
1324                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1325                 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1326                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1327
1328                 /* Clear Half duplex */
1329                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1330                 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1331                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1332
1333                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1334                 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1335
1336                 return 0;
1337         }
1338
1339         if ((phy_type == NES_PHY_TYPE_IRIS) ||
1340             (phy_type == NES_PHY_TYPE_ARGUS) ||
1341             (phy_type == NES_PHY_TYPE_SFP_D)) {
1342                 /* setup 10G MDIO operation */
1343                 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1344                 tx_config &= 0xFFFFFFE3;
1345                 tx_config |= 0x15;
1346                 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1347         }
1348         if ((phy_type == NES_PHY_TYPE_ARGUS) ||
1349             (phy_type == NES_PHY_TYPE_SFP_D)) {
1350                 /* Check firmware heartbeat */
1351                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1352                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1353                 udelay(1500);
1354                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1355                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1356
1357                 if (temp_phy_data != temp_phy_data2)
1358                         return 0;
1359
1360                 /* no heartbeat, configure the PHY */
1361                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1362                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1363                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1364                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1365                 if (phy_type == NES_PHY_TYPE_ARGUS) {
1366                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1367                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1368                 } else {
1369                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1370                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1371                 }
1372                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1373                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1374                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1375
1376                 /* setup LEDs */
1377                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1378                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1379                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1380
1381                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1382
1383                 /* Bring PHY out of reset */
1384                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1385
1386                 /* Check for heartbeat */
1387                 counter = 0;
1388                 mdelay(690);
1389                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1390                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1391                 do {
1392                         if (counter++ > 150) {
1393                                 nes_debug(NES_DBG_PHY, "No PHY heartbeat\n");
1394                                 break;
1395                         }
1396                         mdelay(1);
1397                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1398                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1399                 } while ((temp_phy_data2 == temp_phy_data));
1400
1401                 /* wait for tracking */
1402                 counter = 0;
1403                 do {
1404                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1405                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1406                         if (counter++ > 300) {
1407                                 nes_debug(NES_DBG_PHY, "PHY did not track\n");
1408                                 break;
1409                         }
1410                         mdelay(10);
1411                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1412
1413                 /* setup signal integrity */
1414                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1417                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1419
1420                 /* reset serdes */
1421                 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1422                                        mac_index * 0x200);
1423                 sds |= 0x1;
1424                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1425                                   mac_index * 0x200, sds);
1426                 sds &= 0xfffffffe;
1427                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1428                                   mac_index * 0x200, sds);
1429
1430                 counter = 0;
1431                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1432                                 && (counter++ < 5000))
1433                         ;
1434         }
1435         return 0;
1436 }
1437
1438
1439 /**
1440  * nes_replenish_nic_rq
1441  */
1442 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1443 {
1444         unsigned long flags;
1445         dma_addr_t bus_address;
1446         struct sk_buff *skb;
1447         struct nes_hw_nic_rq_wqe *nic_rqe;
1448         struct nes_hw_nic *nesnic;
1449         struct nes_device *nesdev;
1450         u32 rx_wqes_posted = 0;
1451
1452         nesnic = &nesvnic->nic;
1453         nesdev = nesvnic->nesdev;
1454         spin_lock_irqsave(&nesnic->rq_lock, flags);
1455         if (nesnic->replenishing_rq !=0) {
1456                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1457                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1458                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1459                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1460                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1461                         add_timer(&nesvnic->rq_wqes_timer);
1462                 } else
1463                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1464                 return;
1465         }
1466         nesnic->replenishing_rq = 1;
1467         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1468         do {
1469                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1470                 if (skb) {
1471                         skb->dev = nesvnic->netdev;
1472
1473                         bus_address = pci_map_single(nesdev->pcidev,
1474                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1475
1476                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1477                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1478                                         cpu_to_le32(nesvnic->max_frame_size);
1479                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1480                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1481                                         cpu_to_le32((u32)bus_address);
1482                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1483                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1484                         nesnic->rx_skb[nesnic->rq_head] = skb;
1485                         nesnic->rq_head++;
1486                         nesnic->rq_head &= nesnic->rq_size - 1;
1487                         atomic_dec(&nesvnic->rx_skbs_needed);
1488                         barrier();
1489                         if (++rx_wqes_posted == 255) {
1490                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1491                                 rx_wqes_posted = 0;
1492                         }
1493                 } else {
1494                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1495                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1496                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1497                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1498                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1499                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1500                                 add_timer(&nesvnic->rq_wqes_timer);
1501                         } else
1502                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1503                         break;
1504                 }
1505         } while (atomic_read(&nesvnic->rx_skbs_needed));
1506         barrier();
1507         if (rx_wqes_posted)
1508                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1509         nesnic->replenishing_rq = 0;
1510 }
1511
1512
1513 /**
1514  * nes_rq_wqes_timeout
1515  */
1516 static void nes_rq_wqes_timeout(unsigned long parm)
1517 {
1518         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1519         printk("%s: Timer fired.\n", __func__);
1520         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1521         if (atomic_read(&nesvnic->rx_skbs_needed))
1522                 nes_replenish_nic_rq(nesvnic);
1523 }
1524
1525
1526 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1527                                void **tcph, u64 *hdr_flags, void *priv)
1528 {
1529         unsigned int ip_len;
1530         struct iphdr *iph;
1531         skb_reset_network_header(skb);
1532         iph = ip_hdr(skb);
1533         if (iph->protocol != IPPROTO_TCP)
1534                 return -1;
1535         ip_len = ip_hdrlen(skb);
1536         skb_set_transport_header(skb, ip_len);
1537         *tcph = tcp_hdr(skb);
1538
1539         *hdr_flags = LRO_IPV4 | LRO_TCP;
1540         *iphdr = iph;
1541         return 0;
1542 }
1543
1544
1545 /**
1546  * nes_init_nic_qp
1547  */
1548 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1549 {
1550         struct nes_hw_cqp_wqe *cqp_wqe;
1551         struct nes_hw_nic_sq_wqe *nic_sqe;
1552         struct nes_hw_nic_qp_context *nic_context;
1553         struct sk_buff *skb;
1554         struct nes_hw_nic_rq_wqe *nic_rqe;
1555         struct nes_vnic *nesvnic = netdev_priv(netdev);
1556         unsigned long flags;
1557         void *vmem;
1558         dma_addr_t pmem;
1559         u64 u64temp;
1560         int ret;
1561         u32 cqp_head;
1562         u32 counter;
1563         u32 wqe_count;
1564         u8 jumbomode=0;
1565
1566         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1567         nesvnic->nic_mem_size = 256 +
1568                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1569                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1570                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1571                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1572                         sizeof(struct nes_hw_nic_qp_context);
1573
1574         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1575                         &nesvnic->nic_pbase);
1576         if (!nesvnic->nic_vbase) {
1577                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1578                 return -ENOMEM;
1579         }
1580         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1581         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1582                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1583
1584         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1585                         ~(unsigned long)(256 - 1));
1586         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1587                         ~(unsigned long long)(256 - 1));
1588
1589         /* Setup the first Fragment buffers */
1590         nesvnic->nic.first_frag_vbase = vmem;
1591
1592         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1593                 nesvnic->nic.frag_paddr[counter] = pmem;
1594                 pmem += sizeof(struct nes_first_frag);
1595         }
1596
1597         /* setup the SQ */
1598         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1599
1600         nesvnic->nic.sq_vbase = (void *)vmem;
1601         nesvnic->nic.sq_pbase = pmem;
1602         nesvnic->nic.sq_head = 0;
1603         nesvnic->nic.sq_tail = 0;
1604         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1605         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1606                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1607                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1608                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1609                                 NES_NIC_SQ_WQE_COMPLETION);
1610                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1611                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1612                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1613                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1614                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1615                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1616         }
1617
1618         nesvnic->get_cqp_request = nes_get_cqp_request;
1619         nesvnic->post_cqp_request = nes_post_cqp_request;
1620         nesvnic->mcrq_mcast_filter = NULL;
1621
1622         spin_lock_init(&nesvnic->nic.rq_lock);
1623
1624         /* setup the RQ */
1625         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1626         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1627
1628
1629         nesvnic->nic.rq_vbase = vmem;
1630         nesvnic->nic.rq_pbase = pmem;
1631         nesvnic->nic.rq_head = 0;
1632         nesvnic->nic.rq_tail = 0;
1633         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1634
1635         /* setup the CQ */
1636         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1637         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1638
1639         if (nesdev->nesadapter->netdev_count > 2)
1640                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1641         else
1642                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1643
1644         nesvnic->nic_cq.cq_vbase = vmem;
1645         nesvnic->nic_cq.cq_pbase = pmem;
1646         nesvnic->nic_cq.cq_head = 0;
1647         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1648
1649         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1650
1651         /* Send CreateCQ request to CQP */
1652         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1653         cqp_head = nesdev->cqp.sq_head;
1654
1655         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1656         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1657
1658         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1659                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1660                         ((u32)nesvnic->nic_cq.cq_size << 16));
1661         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1662                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1663         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1664         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1665         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1666         u64temp = (unsigned long)&nesvnic->nic_cq;
1667         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1668         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1669                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1670         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1671         if (++cqp_head >= nesdev->cqp.sq_size)
1672                 cqp_head = 0;
1673         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1674         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1675
1676         /* Send CreateQP request to CQP */
1677         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1678         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1679                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1680                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1681         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1682                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1683                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1684         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1685                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1686         }
1687
1688         u64temp = (u64)nesvnic->nic.sq_pbase;
1689         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1690         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1691         u64temp = (u64)nesvnic->nic.rq_pbase;
1692         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1693         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1694
1695         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1696                         NES_CQP_QP_TYPE_NIC);
1697         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1698         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1699                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1700         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1701
1702         if (++cqp_head >= nesdev->cqp.sq_size)
1703                 cqp_head = 0;
1704         nesdev->cqp.sq_head = cqp_head;
1705
1706         barrier();
1707
1708         /* Ring doorbell (2 WQEs) */
1709         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1710
1711         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1712         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1713                         nesvnic->nic.qp_id);
1714
1715         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1716                         NES_EVENT_TIMEOUT);
1717         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1718                         nesvnic->nic.qp_id, ret);
1719         if (!ret) {
1720                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1721                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1722                                 nesvnic->nic_pbase);
1723                 return -EIO;
1724         }
1725
1726         /* Populate the RQ */
1727         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1728                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1729                 if (!skb) {
1730                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1731
1732                         nes_destroy_nic_qp(nesvnic);
1733                         return -ENOMEM;
1734                 }
1735
1736                 skb->dev = netdev;
1737
1738                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1739                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1740
1741                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1742                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1743                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1744                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1745                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1746                 nesvnic->nic.rx_skb[counter] = skb;
1747         }
1748
1749         wqe_count = NES_NIC_WQ_SIZE - 1;
1750         nesvnic->nic.rq_head = wqe_count;
1751         barrier();
1752         do {
1753                 counter = min(wqe_count, ((u32)255));
1754                 wqe_count -= counter;
1755                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1756         } while (wqe_count);
1757         init_timer(&nesvnic->rq_wqes_timer);
1758         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1759         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1760         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1761         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1762         {
1763                 nes_nic_init_timer(nesdev);
1764                 if (netdev->mtu > 1500)
1765                         jumbomode = 1;
1766                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1767         }
1768         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1769         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1770         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1771         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1772         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1773         nesvnic->lro_mgr.dev            = netdev;
1774         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1775         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1776         return 0;
1777 }
1778
1779
1780 /**
1781  * nes_destroy_nic_qp
1782  */
1783 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1784 {
1785         u64 u64temp;
1786         dma_addr_t bus_address;
1787         struct nes_device *nesdev = nesvnic->nesdev;
1788         struct nes_hw_cqp_wqe *cqp_wqe;
1789         struct nes_hw_nic_sq_wqe *nic_sqe;
1790         struct nes_hw_nic_rq_wqe *nic_rqe;
1791         __le16 *wqe_fragment_length;
1792         u16  wqe_fragment_index;
1793         u64 wqe_frag;
1794         u32 cqp_head;
1795         unsigned long flags;
1796         int ret;
1797
1798         /* Free remaining NIC receive buffers */
1799         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1800                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1801                 wqe_frag  = (u64)le32_to_cpu(
1802                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1803                 wqe_frag |= ((u64)le32_to_cpu(
1804                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1805                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1806                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1807                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1808                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1809         }
1810
1811         /* Free remaining NIC transmit buffers */
1812         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1813                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1814                 wqe_fragment_index = 1;
1815                 wqe_fragment_length = (__le16 *)
1816                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1817                 /* bump past the vlan tag */
1818                 wqe_fragment_length++;
1819                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1820                         u64temp = (u64)le32_to_cpu(
1821                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1822                                 wqe_fragment_index*2]);
1823                         u64temp += ((u64)le32_to_cpu(
1824                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1825                                 + wqe_fragment_index*2]))<<32;
1826                         bus_address = (dma_addr_t)u64temp;
1827                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1828                                         nesvnic->nic.first_frag_overflow)) {
1829                                 pci_unmap_single(nesdev->pcidev,
1830                                                 bus_address,
1831                                                 le16_to_cpu(wqe_fragment_length[
1832                                                         wqe_fragment_index++]),
1833                                                 PCI_DMA_TODEVICE);
1834                         }
1835                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1836                                 if (wqe_fragment_length[wqe_fragment_index]) {
1837                                         u64temp = le32_to_cpu(
1838                                                 nic_sqe->wqe_words[
1839                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1840                                                 wqe_fragment_index*2]);
1841                                         u64temp += ((u64)le32_to_cpu(
1842                                                 nic_sqe->wqe_words[
1843                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1844                                                 wqe_fragment_index*2]))<<32;
1845                                         bus_address = (dma_addr_t)u64temp;
1846                                         pci_unmap_page(nesdev->pcidev,
1847                                                         bus_address,
1848                                                         le16_to_cpu(
1849                                                         wqe_fragment_length[
1850                                                         wqe_fragment_index]),
1851                                                         PCI_DMA_TODEVICE);
1852                                 } else
1853                                         break;
1854                         }
1855                 }
1856                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1857                         dev_kfree_skb(
1858                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1859
1860                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1861                                         & (nesvnic->nic.sq_size - 1);
1862         }
1863
1864         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1865
1866         /* Destroy NIC QP */
1867         cqp_head = nesdev->cqp.sq_head;
1868         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1869         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1870
1871         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1872                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1873         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1874                 nesvnic->nic.qp_id);
1875
1876         if (++cqp_head >= nesdev->cqp.sq_size)
1877                 cqp_head = 0;
1878
1879         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1880
1881         /* Destroy NIC CQ */
1882         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1883         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1884                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1885         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1886                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1887
1888         if (++cqp_head >= nesdev->cqp.sq_size)
1889                 cqp_head = 0;
1890
1891         nesdev->cqp.sq_head = cqp_head;
1892         barrier();
1893
1894         /* Ring doorbell (2 WQEs) */
1895         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1896
1897         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1898         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1899                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1900                         cqp_head, nesdev->cqp.sq_head,
1901                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1902
1903         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1904                         NES_EVENT_TIMEOUT);
1905
1906         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1907                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1908                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1909         if (!ret) {
1910                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1911                                 nesvnic->nic.qp_id);
1912         }
1913
1914         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1915                         nesvnic->nic_pbase);
1916 }
1917
1918 /**
1919  * nes_napi_isr
1920  */
1921 int nes_napi_isr(struct nes_device *nesdev)
1922 {
1923         struct nes_adapter *nesadapter = nesdev->nesadapter;
1924         u32 int_stat;
1925
1926         if (nesdev->napi_isr_ran) {
1927                 /* interrupt status has already been read in ISR */
1928                 int_stat = nesdev->int_stat;
1929         } else {
1930                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1931                 nesdev->int_stat = int_stat;
1932                 nesdev->napi_isr_ran = 1;
1933         }
1934
1935         int_stat &= nesdev->int_req;
1936         /* iff NIC, process here, else wait for DPC */
1937         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1938                 nesdev->napi_isr_ran = 0;
1939                 nes_write32(nesdev->regs + NES_INT_STAT,
1940                         (int_stat &
1941                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1942
1943                 /* Process the CEQs */
1944                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1945
1946                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1947                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1948                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1949                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1950                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1951                                 /* Enable Periodic timer interrupts */
1952                                 nesdev->int_req |= NES_INT_TIMER;
1953                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1954                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1955                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1956                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1957                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1958                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1959                         }
1960
1961                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1962                         {
1963                                 nes_nic_init_timer(nesdev);
1964                         }
1965                         /* Enable interrupts, except CEQs */
1966                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1967                 } else {
1968                         /* Enable interrupts, make sure timer is off */
1969                         nesdev->int_req &= ~NES_INT_TIMER;
1970                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1971                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1972                 }
1973                 nesdev->deepcq_count = 0;
1974                 return 1;
1975         } else {
1976                 return 0;
1977         }
1978 }
1979
1980 static void process_critical_error(struct nes_device *nesdev)
1981 {
1982         u32 debug_error;
1983         u32 nes_idx_debug_error_masks0 = 0;
1984         u16 error_module = 0;
1985
1986         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1987         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1988                         (u16)debug_error);
1989         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1990                         0x01010000 | (debug_error & 0x0000ffff));
1991         if (crit_err_count++ > 10)
1992                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1993         error_module = (u16) (debug_error & 0x1F00) >> 8;
1994         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
1995                         nes_max_critical_error_count) {
1996                 printk(KERN_ERR PFX "Masking off critical error for module "
1997                         "0x%02X\n", (u16)error_module);
1998                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
1999                         NES_IDX_DEBUG_ERROR_MASKS0);
2000                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2001                         nes_idx_debug_error_masks0 | (1 << error_module));
2002         }
2003 }
2004 /**
2005  * nes_dpc
2006  */
2007 void nes_dpc(unsigned long param)
2008 {
2009         struct nes_device *nesdev = (struct nes_device *)param;
2010         struct nes_adapter *nesadapter = nesdev->nesadapter;
2011         u32 counter;
2012         u32 loop_counter = 0;
2013         u32 int_status_bit;
2014         u32 int_stat;
2015         u32 timer_stat;
2016         u32 temp_int_stat;
2017         u32 intf_int_stat;
2018         u32 processed_intf_int = 0;
2019         u16 processed_timer_int = 0;
2020         u16 completion_ints = 0;
2021         u16 timer_ints = 0;
2022
2023         /* nes_debug(NES_DBG_ISR, "\n"); */
2024
2025         do {
2026                 timer_stat = 0;
2027                 if (nesdev->napi_isr_ran) {
2028                         nesdev->napi_isr_ran = 0;
2029                         int_stat = nesdev->int_stat;
2030                 } else
2031                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2032                 if (processed_intf_int != 0)
2033                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2034                 else
2035                         int_stat &= nesdev->int_req;
2036                 if (processed_timer_int == 0) {
2037                         processed_timer_int = 1;
2038                         if (int_stat & NES_INT_TIMER) {
2039                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2040                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2041                                         int_stat &= ~NES_INT_TIMER;
2042                                 }
2043                         }
2044                 } else {
2045                         int_stat &= ~NES_INT_TIMER;
2046                 }
2047
2048                 if (int_stat) {
2049                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2050                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2051                                 /* Ack the interrupts */
2052                                 nes_write32(nesdev->regs+NES_INT_STAT,
2053                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2054                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2055                         }
2056
2057                         temp_int_stat = int_stat;
2058                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2059                                 if (int_stat & int_status_bit) {
2060                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2061                                         temp_int_stat &= ~int_status_bit;
2062                                         completion_ints = 1;
2063                                 }
2064                                 if (!(temp_int_stat & 0x0000ffff))
2065                                         break;
2066                                 int_status_bit <<= 1;
2067                         }
2068
2069                         /* Process the AEQ for this pci function */
2070                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2071                         if (int_stat & int_status_bit) {
2072                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2073                         }
2074
2075                         /* Process the MAC interrupt for this pci function */
2076                         int_status_bit = 1 << (24 + nesdev->mac_index);
2077                         if (int_stat & int_status_bit) {
2078                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2079                         }
2080
2081                         if (int_stat & NES_INT_TIMER) {
2082                                 if (timer_stat & nesdev->timer_int_req) {
2083                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2084                                                         (timer_stat & nesdev->timer_int_req) |
2085                                                         ~(nesdev->nesadapter->timer_int_req));
2086                                         timer_ints = 1;
2087                                 }
2088                         }
2089
2090                         if (int_stat & NES_INT_INTF) {
2091                                 processed_intf_int = 1;
2092                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2093                                 intf_int_stat &= nesdev->intf_int_req;
2094                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2095                                         process_critical_error(nesdev);
2096                                 }
2097                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2098                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2099                                         BUG();
2100                                 }
2101                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2102                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2103                                         BUG();
2104                                 }
2105                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2106                         }
2107
2108                         if (int_stat & NES_INT_TSW) {
2109                         }
2110                 }
2111                 /* Don't use the interface interrupt bit stay in loop */
2112                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2113                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2114         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2115
2116         if (timer_ints == 1) {
2117                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2118                         if (completion_ints == 0) {
2119                                 nesdev->timer_only_int_count++;
2120                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2121                                         nesdev->timer_only_int_count = 0;
2122                                         nesdev->int_req &= ~NES_INT_TIMER;
2123                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2124                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2125                                 } else {
2126                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2127                                 }
2128                         } else {
2129                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2130                                 {
2131                                         nes_nic_init_timer(nesdev);
2132                                 }
2133                                 nesdev->timer_only_int_count = 0;
2134                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2135                         }
2136                 } else {
2137                         nesdev->timer_only_int_count = 0;
2138                         nesdev->int_req &= ~NES_INT_TIMER;
2139                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2140                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2141                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2142                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2143                 }
2144         } else {
2145                 if ( (completion_ints == 1) &&
2146                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2147                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2148                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2149                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2150                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2151                         nesdev->timer_only_int_count = 0;
2152                         nesdev->int_req |= NES_INT_TIMER;
2153                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2154                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2155                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2156                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2157                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2158                 } else {
2159                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2160                 }
2161         }
2162         nesdev->deepcq_count = 0;
2163 }
2164
2165
2166 /**
2167  * nes_process_ceq
2168  */
2169 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2170 {
2171         u64 u64temp;
2172         struct nes_hw_cq *cq;
2173         u32 head;
2174         u32 ceq_size;
2175
2176         /* nes_debug(NES_DBG_CQ, "\n"); */
2177         head = ceq->ceq_head;
2178         ceq_size = ceq->ceq_size;
2179
2180         do {
2181                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2182                                 NES_CEQE_VALID) {
2183                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2184                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2185                         u64temp <<= 1;
2186                         cq = *((struct nes_hw_cq **)&u64temp);
2187                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2188                         barrier();
2189                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2190
2191                         /* call the event handler */
2192                         cq->ce_handler(nesdev, cq);
2193
2194                         if (++head >= ceq_size)
2195                                 head = 0;
2196                 } else {
2197                         break;
2198                 }
2199
2200         } while (1);
2201
2202         ceq->ceq_head = head;
2203 }
2204
2205
2206 /**
2207  * nes_process_aeq
2208  */
2209 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2210 {
2211         /* u64 u64temp; */
2212         u32 head;
2213         u32 aeq_size;
2214         u32 aeqe_misc;
2215         u32 aeqe_cq_id;
2216         struct nes_hw_aeqe volatile *aeqe;
2217
2218         head = aeq->aeq_head;
2219         aeq_size = aeq->aeq_size;
2220
2221         do {
2222                 aeqe = &aeq->aeq_vbase[head];
2223                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2224                         break;
2225                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2226                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2227                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2228                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2229                                 /* dealing with an accelerated QP related AE */
2230                                 /*
2231                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2232                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2233                                  */
2234                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2235                         } else {
2236                                 /* TODO: dealing with a CQP related AE */
2237                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2238                                                 (u16)(aeqe_misc >> 16));
2239                         }
2240                 }
2241
2242                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2243
2244                 if (++head >= aeq_size)
2245                         head = 0;
2246
2247                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2248         }
2249         while (1);
2250         aeq->aeq_head = head;
2251 }
2252
2253 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2254 {
2255         struct nes_adapter *nesadapter = nesdev->nesadapter;
2256         u32 reset_value;
2257         u32 i=0;
2258         u32 u32temp;
2259
2260         if (nesadapter->hw_rev == NE020_REV) {
2261                 return;
2262         }
2263         mh_detected++;
2264
2265         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2266
2267         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2268                 reset_value |= 0x0000001d;
2269         else
2270                 reset_value |= 0x0000002d;
2271
2272         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2273                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2274                         nesadapter->link_interrupt_count[0] = 0;
2275                         nesadapter->link_interrupt_count[1] = 0;
2276                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2277                         if (0x00000040 & u32temp)
2278                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2279                         else
2280                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2281
2282                         reset_value |= 0x0000003d;
2283                 }
2284                 nesadapter->link_interrupt_count[mac_index] = 0;
2285         }
2286
2287         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2288
2289         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2290                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2291
2292         if (0x0000003d == (reset_value & 0x0000003d)) {
2293                 u32 pcs_control_status0, pcs_control_status1;
2294
2295                 for (i = 0; i < 10; i++) {
2296                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2297                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2298                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2299                              && (pcs_control_status0 & 0x00100000))
2300                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2301                                 && (pcs_control_status1 & 0x00100000)))
2302                                 continue;
2303                         else
2304                                 break;
2305                 }
2306                 if (10 == i) {
2307                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2308                         if (0x00000040 & u32temp)
2309                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2310                         else
2311                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2312
2313                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2314
2315                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2316                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2317                 }
2318         }
2319 }
2320
2321 /**
2322  * nes_process_mac_intr
2323  */
2324 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2325 {
2326         unsigned long flags;
2327         u32 pcs_control_status;
2328         struct nes_adapter *nesadapter = nesdev->nesadapter;
2329         struct nes_vnic *nesvnic;
2330         u32 mac_status;
2331         u32 mac_index = nesdev->mac_index;
2332         u32 u32temp;
2333         u16 phy_data;
2334         u16 temp_phy_data;
2335         u32 pcs_val  = 0x0f0f0000;
2336         u32 pcs_mask = 0x0f1f0000;
2337         u32 cdr_ctrl;
2338
2339         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2340         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2341                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2342                 return;
2343         }
2344         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2345         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2346
2347         /* ack the MAC interrupt */
2348         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2349         /* Clear the interrupt */
2350         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2351
2352         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2353
2354         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2355                 nesdev->link_status_interrupts++;
2356                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2357                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2358                         nes_reset_link(nesdev, mac_index);
2359                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2360                 }
2361                 /* read the PHY interrupt status register */
2362                 if ((nesadapter->OneG_Mode) &&
2363                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2364                         do {
2365                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2366                                                 nesadapter->phy_index[mac_index], &phy_data);
2367                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2368                                                 nesadapter->phy_index[mac_index], phy_data);
2369                         } while (phy_data&0x8000);
2370
2371                         temp_phy_data = 0;
2372                         do {
2373                                 nes_read_1G_phy_reg(nesdev, 0x11,
2374                                                 nesadapter->phy_index[mac_index], &phy_data);
2375                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2376                                                 nesadapter->phy_index[mac_index], phy_data);
2377                                 if (temp_phy_data == phy_data)
2378                                         break;
2379                                 temp_phy_data = phy_data;
2380                         } while (1);
2381
2382                         nes_read_1G_phy_reg(nesdev, 0x1e,
2383                                         nesadapter->phy_index[mac_index], &phy_data);
2384                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2385                                         nesadapter->phy_index[mac_index], phy_data);
2386
2387                         nes_read_1G_phy_reg(nesdev, 1,
2388                                         nesadapter->phy_index[mac_index], &phy_data);
2389                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2390                                         nesadapter->phy_index[mac_index], phy_data);
2391
2392                         if (temp_phy_data & 0x1000) {
2393                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2394                                 phy_data = 4;
2395                         } else {
2396                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2397                         }
2398                 }
2399                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2400                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2401                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2402
2403                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2404                         switch (mac_index) {
2405                         case 1:
2406                         case 3:
2407                                 pcs_control_status = nes_read_indexed(nesdev,
2408                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2409                                 break;
2410                         default:
2411                                 pcs_control_status = nes_read_indexed(nesdev,
2412                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2413                                 break;
2414                         }
2415                 } else {
2416                         pcs_control_status = nes_read_indexed(nesdev,
2417                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2418                         pcs_control_status = nes_read_indexed(nesdev,
2419                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2420                 }
2421
2422                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2423                                 mac_index, pcs_control_status);
2424                 if ((nesadapter->OneG_Mode) &&
2425                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2426                         u32temp = 0x01010000;
2427                         if (nesadapter->port_count > 2) {
2428                                 u32temp |= 0x02020000;
2429                         }
2430                         if ((pcs_control_status & u32temp)!= u32temp) {
2431                                 phy_data = 0;
2432                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2433                         }
2434                 } else {
2435                         switch (nesadapter->phy_type[mac_index]) {
2436                         case NES_PHY_TYPE_IRIS:
2437                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2438                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2439                                 u32temp = 20;
2440                                 do {
2441                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2442                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2443                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2444                                                 break;
2445                                         temp_phy_data = phy_data;
2446                                 } while (1);
2447                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2448                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2449                                 break;
2450
2451                         case NES_PHY_TYPE_ARGUS:
2452                         case NES_PHY_TYPE_SFP_D:
2453                                 /* clear the alarms */
2454                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2455                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2456                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2457                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2458                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2459                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2460                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2461                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2462                                 /* check link status */
2463                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2464                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2465
2466                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2467                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2468                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2469                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2470
2471                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2472
2473                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2474                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2475                                 break;
2476
2477                         case NES_PHY_TYPE_PUMA_1G:
2478                                 if (mac_index < 2)
2479                                         pcs_val = pcs_mask = 0x01010000;
2480                                 else
2481                                         pcs_val = pcs_mask = 0x02020000;
2482                                 /* fall through */
2483                         default:
2484                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2485                                 break;
2486                         }
2487                 }
2488
2489                 if (phy_data & 0x0004) {
2490                         if (wide_ppm_offset &&
2491                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2492                             (nesadapter->hw_rev != NE020_REV)) {
2493                                 cdr_ctrl = nes_read_indexed(nesdev,
2494                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2495                                                             mac_index * 0x200);
2496                                 nes_write_indexed(nesdev,
2497                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2498                                                   mac_index * 0x200,
2499                                                   cdr_ctrl | 0x000F0000);
2500                         }
2501                         nesadapter->mac_link_down[mac_index] = 0;
2502                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2503                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2504                                                 nesvnic->linkup);
2505                                 if (nesvnic->linkup == 0) {
2506                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2507                                                         nesvnic->netdev->name, nesvnic->netdev);
2508                                         if (netif_queue_stopped(nesvnic->netdev))
2509                                                 netif_start_queue(nesvnic->netdev);
2510                                         nesvnic->linkup = 1;
2511                                         netif_carrier_on(nesvnic->netdev);
2512                                 }
2513                         }
2514                 } else {
2515                         if (wide_ppm_offset &&
2516                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2517                             (nesadapter->hw_rev != NE020_REV)) {
2518                                 cdr_ctrl = nes_read_indexed(nesdev,
2519                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2520                                                             mac_index * 0x200);
2521                                 nes_write_indexed(nesdev,
2522                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2523                                                   mac_index * 0x200,
2524                                                   cdr_ctrl & 0xFFF0FFFF);
2525                         }
2526                         nesadapter->mac_link_down[mac_index] = 1;
2527                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2528                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2529                                                 nesvnic->linkup);
2530                                 if (nesvnic->linkup == 1) {
2531                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2532                                                         nesvnic->netdev->name, nesvnic->netdev);
2533                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2534                                                 netif_stop_queue(nesvnic->netdev);
2535                                         nesvnic->linkup = 0;
2536                                         netif_carrier_off(nesvnic->netdev);
2537                                 }
2538                         }
2539                 }
2540         }
2541
2542         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2543 }
2544
2545
2546
2547 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2548 {
2549         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2550
2551         napi_schedule(&nesvnic->napi);
2552 }
2553
2554
2555 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2556 * getting out of nic_ce_handler
2557 */
2558 #define MAX_RQES_TO_PROCESS     384
2559
2560 /**
2561  * nes_nic_ce_handler
2562  */
2563 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2564 {
2565         u64 u64temp;
2566         dma_addr_t bus_address;
2567         struct nes_hw_nic *nesnic;
2568         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2569         struct nes_adapter *nesadapter = nesdev->nesadapter;
2570         struct nes_hw_nic_rq_wqe *nic_rqe;
2571         struct nes_hw_nic_sq_wqe *nic_sqe;
2572         struct sk_buff *skb;
2573         struct sk_buff *rx_skb;
2574         __le16 *wqe_fragment_length;
2575         u32 head;
2576         u32 cq_size;
2577         u32 rx_pkt_size;
2578         u32 cqe_count=0;
2579         u32 cqe_errv;
2580         u32 cqe_misc;
2581         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2582         u16 vlan_tag;
2583         u16 pkt_type;
2584         u16 rqes_processed = 0;
2585         u8 sq_cqes = 0;
2586         u8 nes_use_lro = 0;
2587
2588         head = cq->cq_head;
2589         cq_size = cq->cq_size;
2590         cq->cqes_pending = 1;
2591         if (nesvnic->netdev->features & NETIF_F_LRO)
2592                 nes_use_lro = 1;
2593         do {
2594                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2595                                 NES_NIC_CQE_VALID) {
2596                         nesnic = &nesvnic->nic;
2597                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2598                         if (cqe_misc & NES_NIC_CQE_SQ) {
2599                                 sq_cqes++;
2600                                 wqe_fragment_index = 1;
2601                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2602                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2603                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2604                                 /* bump past the vlan tag */
2605                                 wqe_fragment_length++;
2606                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2607                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2608                                                         wqe_fragment_index * 2]);
2609                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2610                                                         wqe_fragment_index * 2])) << 32;
2611                                         bus_address = (dma_addr_t)u64temp;
2612                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2613                                                 pci_unmap_single(nesdev->pcidev,
2614                                                                 bus_address,
2615                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2616                                                                 PCI_DMA_TODEVICE);
2617                                         }
2618                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2619                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2620                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2621                                                                                 wqe_fragment_index * 2]);
2622                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2623                                                                                 + wqe_fragment_index * 2])) <<32;
2624                                                         bus_address = (dma_addr_t)u64temp;
2625                                                         pci_unmap_page(nesdev->pcidev,
2626                                                                         bus_address,
2627                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2628                                                                         PCI_DMA_TODEVICE);
2629                                                 } else
2630                                                         break;
2631                                         }
2632                                 }
2633                                 if (skb)
2634                                         dev_kfree_skb_any(skb);
2635                                 nesnic->sq_tail++;
2636                                 nesnic->sq_tail &= nesnic->sq_size-1;
2637                                 if (sq_cqes > 128) {
2638                                         barrier();
2639                                 /* restart the queue if it had been stopped */
2640                                 if (netif_queue_stopped(nesvnic->netdev))
2641                                         netif_wake_queue(nesvnic->netdev);
2642                                         sq_cqes = 0;
2643                                 }
2644                         } else {
2645                                 rqes_processed ++;
2646
2647                                 cq->rx_cqes_completed++;
2648                                 cq->rx_pkts_indicated++;
2649                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2650                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2651                                 /* Get the skb */
2652                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2653                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2654                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2655                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2656                                 pci_unmap_single(nesdev->pcidev, bus_address,
2657                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2658                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2659                                 /* rx_skb->len = rx_pkt_size; */
2660                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2661                                 skb_put(rx_skb, rx_pkt_size);
2662                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2663                                 nesnic->rq_tail++;
2664                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2665
2666                                 atomic_inc(&nesvnic->rx_skbs_needed);
2667                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2668                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2669                                                         cq->cq_number | (cqe_count << 16));
2670                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2671                                         nesdev->currcq_count += cqe_count;
2672                                         cqe_count = 0;
2673                                         nes_replenish_nic_rq(nesvnic);
2674                                 }
2675                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2676                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2677                                 rx_skb->ip_summed = CHECKSUM_NONE;
2678
2679                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2680                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2681                                         if ((cqe_errv &
2682                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2683                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2684                                                 if (nesvnic->rx_checksum_disabled == 0) {
2685                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2686                                                 }
2687                                         } else
2688                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2689                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2690                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2691
2692                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2693                                         if ((cqe_errv &
2694                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2695                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2696                                                 if (nesvnic->rx_checksum_disabled == 0) {
2697                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2698                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2699                                                                   nesvnic->netdev->name); */
2700                                                 }
2701                                         } else
2702                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2703                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2704                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2705                                         }
2706                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2707                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2708
2709                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2710                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2711                                                 rx_skb = NULL;
2712                                 }
2713                                 if (rx_skb == NULL)
2714                                         goto skip_rx_indicate0;
2715
2716
2717                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2718                                     (nesvnic->vlan_grp != NULL)) {
2719                                         vlan_tag = (u16)(le32_to_cpu(
2720                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2721                                                         >> 16);
2722                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2723                                                         nesvnic->netdev->name, vlan_tag);
2724                                         if (nes_use_lro)
2725                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2726                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2727                                         else
2728                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2729                                 } else {
2730                                         if (nes_use_lro)
2731                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2732                                         else
2733                                                 nes_netif_rx(rx_skb);
2734                                 }
2735
2736 skip_rx_indicate0:
2737                                 nesvnic->netdev->last_rx = jiffies;
2738                                 /* nesvnic->netstats.rx_packets++; */
2739                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2740                         }
2741
2742                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2743                         /* Accounting... */
2744                         cqe_count++;
2745                         if (++head >= cq_size)
2746                                 head = 0;
2747                         if (cqe_count == 255) {
2748                                 /* Replenish Nic CQ */
2749                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2750                                                 cq->cq_number | (cqe_count << 16));
2751                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2752                                 nesdev->currcq_count += cqe_count;
2753                                 cqe_count = 0;
2754                         }
2755
2756                         if (cq->rx_cqes_completed >= nesvnic->budget)
2757                                 break;
2758                 } else {
2759                         cq->cqes_pending = 0;
2760                         break;
2761                 }
2762
2763         } while (1);
2764
2765         if (nes_use_lro)
2766                 lro_flush_all(&nesvnic->lro_mgr);
2767         if (sq_cqes) {
2768                 barrier();
2769                 /* restart the queue if it had been stopped */
2770                 if (netif_queue_stopped(nesvnic->netdev))
2771                         netif_wake_queue(nesvnic->netdev);
2772         }
2773         cq->cq_head = head;
2774         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2775                         cq->cq_number, cqe_count, cq->cq_head); */
2776         cq->cqe_allocs_pending = cqe_count;
2777         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2778         {
2779                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2780                 nesdev->currcq_count += cqe_count;
2781                 nes_nic_tune_timer(nesdev);
2782         }
2783         if (atomic_read(&nesvnic->rx_skbs_needed))
2784                 nes_replenish_nic_rq(nesvnic);
2785 }
2786
2787
2788 /**
2789  * nes_cqp_ce_handler
2790  */
2791 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2792 {
2793         u64 u64temp;
2794         unsigned long flags;
2795         struct nes_hw_cqp *cqp = NULL;
2796         struct nes_cqp_request *cqp_request;
2797         struct nes_hw_cqp_wqe *cqp_wqe;
2798         u32 head;
2799         u32 cq_size;
2800         u32 cqe_count=0;
2801         u32 error_code;
2802         /* u32 counter; */
2803
2804         head = cq->cq_head;
2805         cq_size = cq->cq_size;
2806
2807         do {
2808                 /* process the CQE */
2809                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2810                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2811
2812                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2813                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2814                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2815                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2816                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2817                         cqp = *((struct nes_hw_cqp **)&u64temp);
2818
2819                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2820                         if (error_code) {
2821                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2822                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2823                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2824                                                 (u16)(error_code >> 16),
2825                                                 (u16)error_code);
2826                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2827                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2828                         }
2829
2830                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2831                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2832                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2833                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2834                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2835                         if (cqp_request) {
2836                                 if (cqp_request->waiting) {
2837                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2838                                         cqp_request->major_code = (u16)(error_code >> 16);
2839                                         cqp_request->minor_code = (u16)error_code;
2840                                         barrier();
2841                                         cqp_request->request_done = 1;
2842                                         wake_up(&cqp_request->waitq);
2843                                         nes_put_cqp_request(nesdev, cqp_request);
2844                                 } else {
2845                                         if (cqp_request->callback)
2846                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2847                                         nes_free_cqp_request(nesdev, cqp_request);
2848                                 }
2849                         } else {
2850                                 wake_up(&nesdev->cqp.waitq);
2851                         }
2852
2853                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2854                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2855                         if (++cqp->sq_tail >= cqp->sq_size)
2856                                 cqp->sq_tail = 0;
2857
2858                         /* Accounting... */
2859                         cqe_count++;
2860                         if (++head >= cq_size)
2861                                 head = 0;
2862                 } else {
2863                         break;
2864                 }
2865         } while (1);
2866         cq->cq_head = head;
2867
2868         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2869         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2870                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2871                         (nesdev->cqp.sq_size - 1)) != 1)) {
2872                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2873                                 struct nes_cqp_request, list);
2874                 list_del_init(&cqp_request->list);
2875                 head = nesdev->cqp.sq_head++;
2876                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2877                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2878                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2879                 barrier();
2880                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2881                         cpu_to_le32((u32)((unsigned long)cqp_request));
2882                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2883                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2884                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2885                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2886                 /* Ring doorbell (1 WQEs) */
2887                 barrier();
2888                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2889         }
2890         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2891
2892         /* Arm the CCQ */
2893         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2894                         cq->cq_number);
2895         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2896 }
2897
2898
2899 /**
2900  * nes_process_iwarp_aeqe
2901  */
2902 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2903                                    struct nes_hw_aeqe *aeqe)
2904 {
2905         u64 context;
2906         u64 aeqe_context = 0;
2907         unsigned long flags;
2908         struct nes_qp *nesqp;
2909         int resource_allocated;
2910         /* struct iw_cm_id *cm_id; */
2911         struct nes_adapter *nesadapter = nesdev->nesadapter;
2912         struct ib_event ibevent;
2913         /* struct iw_cm_event cm_event; */
2914         u32 aeq_info;
2915         u32 next_iwarp_state = 0;
2916         u16 async_event_id;
2917         u8 tcp_state;
2918         u8 iwarp_state;
2919
2920         nes_debug(NES_DBG_AEQ, "\n");
2921         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2922         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2923                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2924                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2925         } else {
2926                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2927                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2928                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2929                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2930                 BUG_ON(!context);
2931         }
2932
2933         async_event_id = (u16)aeq_info;
2934         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2935         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2936         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2937                         " Tcp state = %s, iWARP state = %s\n",
2938                         async_event_id,
2939                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2940                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2941
2942         switch (async_event_id) {
2943                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2944                         nesqp = *((struct nes_qp **)&context);
2945                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2946                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2947                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2948                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2949                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2950                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2951                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2952                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2953                                                 async_event_id, nesqp->last_aeq, tcp_state);
2954                         }
2955                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2956                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2957                                 /* FIN Received but tcp state or IB state moved on,
2958                                                 should expect a close complete */
2959                                 return;
2960                         }
2961                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2962                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2963                 case NES_AEQE_AEID_TERMINATE_SENT:
2964                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2965                 case NES_AEQE_AEID_RESET_SENT:
2966                         nesqp = *((struct nes_qp **)&context);
2967                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2968                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2969                         }
2970                         spin_lock_irqsave(&nesqp->lock, flags);
2971                         nesqp->hw_iwarp_state = iwarp_state;
2972                         nesqp->hw_tcp_state = tcp_state;
2973                         nesqp->last_aeq = async_event_id;
2974
2975                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2976                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2977                                 nesqp->hte_added = 0;
2978                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2979                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2980                                                 nesqp->hwqp.qp_id);
2981                                 nes_hw_modify_qp(nesdev, nesqp,
2982                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2983                                 spin_lock_irqsave(&nesqp->lock, flags);
2984                         }
2985
2986                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2987                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2988                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2989                                 switch (nesqp->hw_iwarp_state) {
2990                                         case NES_AEQE_IWARP_STATE_RTS:
2991                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2992                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2993                                                 break;
2994                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2995                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2996                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2997                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2998                                                         next_iwarp_state |= 0x02000000;
2999                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3000                                                 }
3001                                                 break;
3002                                         default:
3003                                                 next_iwarp_state = 0;
3004                                 }
3005                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3006                                 if (next_iwarp_state) {
3007                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3008                                                         " also added another reference\n",
3009                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3010                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3011                                 }
3012                                 nes_cm_disconn(nesqp);
3013                         } else {
3014                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3015                                         /* FIN Received but ib state not RTS,
3016                                                         close complete will be on its way */
3017                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3018                                         return;
3019                                 }
3020                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3021                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3022                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
3023                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3024                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3025                                                         " also added another reference\n",
3026                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3027                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3028                                 }
3029                                 nes_cm_disconn(nesqp);
3030                         }
3031                         break;
3032                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3033                         nesqp = *((struct nes_qp **)&context);
3034                         spin_lock_irqsave(&nesqp->lock, flags);
3035                         nesqp->hw_iwarp_state = iwarp_state;
3036                         nesqp->hw_tcp_state = tcp_state;
3037                         nesqp->last_aeq = async_event_id;
3038                         spin_unlock_irqrestore(&nesqp->lock, flags);
3039                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
3040                                         " event on QP%u \n  Q2 Data:\n",
3041                                         nesqp->hwqp.qp_id);
3042                         if (nesqp->ibqp.event_handler) {
3043                                 ibevent.device = nesqp->ibqp.device;
3044                                 ibevent.element.qp = &nesqp->ibqp;
3045                                 ibevent.event = IB_EVENT_QP_FATAL;
3046                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3047                         }
3048                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3049                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
3050                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3051                                 nes_cm_disconn(nesqp);
3052                         } else {
3053                                 nesqp->in_disconnect = 0;
3054                                 wake_up(&nesqp->kick_waitq);
3055                         }
3056                         break;
3057                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3058                         nesqp = *((struct nes_qp **)&context);
3059                         spin_lock_irqsave(&nesqp->lock, flags);
3060                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3061                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3062                         nesqp->last_aeq = async_event_id;
3063                         if (nesqp->cm_id) {
3064                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3065                                                 " event on QP%u, remote IP = 0x%08X \n",
3066                                                 nesqp->hwqp.qp_id,
3067                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
3068                         } else {
3069                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3070                                                 " event on QP%u \n",
3071                                                 nesqp->hwqp.qp_id);
3072                         }
3073                         spin_unlock_irqrestore(&nesqp->lock, flags);
3074                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
3075                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3076                         if (nesqp->ibqp.event_handler) {
3077                                 ibevent.device = nesqp->ibqp.device;
3078                                 ibevent.element.qp = &nesqp->ibqp;
3079                                 ibevent.event = IB_EVENT_QP_FATAL;
3080                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3081                         }
3082                         break;
3083                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3084                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
3085                                 nesqp = nesadapter->qp_table[le32_to_cpu(
3086                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3087                         } else {
3088                                 /* TODO: get the actual WQE and mask off wqe index */
3089                                 context &= ~((u64)511);
3090                                 nesqp = *((struct nes_qp **)&context);
3091                         }
3092                         spin_lock_irqsave(&nesqp->lock, flags);
3093                         nesqp->hw_iwarp_state = iwarp_state;
3094                         nesqp->hw_tcp_state = tcp_state;
3095                         nesqp->last_aeq = async_event_id;
3096                         spin_unlock_irqrestore(&nesqp->lock, flags);
3097                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3098                                         nesqp->hwqp.qp_id);
3099                         if (nesqp->ibqp.event_handler) {
3100                                 ibevent.device = nesqp->ibqp.device;
3101                                 ibevent.element.qp = &nesqp->ibqp;
3102                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3103                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3104                         }
3105                         break;
3106                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3107                         nesqp = *((struct nes_qp **)&context);
3108                         spin_lock_irqsave(&nesqp->lock, flags);
3109                         nesqp->hw_iwarp_state = iwarp_state;
3110                         nesqp->hw_tcp_state = tcp_state;
3111                         nesqp->last_aeq = async_event_id;
3112                         spin_unlock_irqrestore(&nesqp->lock, flags);
3113                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3114                                         nesqp->hwqp.qp_id);
3115                         if (nesqp->ibqp.event_handler) {
3116                                 ibevent.device = nesqp->ibqp.device;
3117                                 ibevent.element.qp = &nesqp->ibqp;
3118                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3119                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3120                         }
3121                         break;
3122                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3123                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3124                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3125                         spin_lock_irqsave(&nesqp->lock, flags);
3126                         nesqp->hw_iwarp_state = iwarp_state;
3127                         nesqp->hw_tcp_state = tcp_state;
3128                         nesqp->last_aeq = async_event_id;
3129                         spin_unlock_irqrestore(&nesqp->lock, flags);
3130                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3131                                         " nesqp = %p, AE reported %p\n",
3132                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3133                         if (nesqp->ibqp.event_handler) {
3134                                 ibevent.device = nesqp->ibqp.device;
3135                                 ibevent.element.qp = &nesqp->ibqp;
3136                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3137                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3138                         }
3139                         break;
3140                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3141                         context <<= 1;
3142                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3143                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3144                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3145                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3146                         if (resource_allocated) {
3147                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3148                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3149                         }
3150                         break;
3151                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3152                         nesqp = nesadapter->qp_table[le32_to_cpu(
3153                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3154                         spin_lock_irqsave(&nesqp->lock, flags);
3155                         nesqp->hw_iwarp_state = iwarp_state;
3156                         nesqp->hw_tcp_state = tcp_state;
3157                         nesqp->last_aeq = async_event_id;
3158                         spin_unlock_irqrestore(&nesqp->lock, flags);
3159                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3160                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3161                                         nesqp->hwqp.qp_id);
3162                         if (nesqp->ibqp.event_handler) {
3163                                 ibevent.device = nesqp->ibqp.device;
3164                                 ibevent.element.qp = &nesqp->ibqp;
3165                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3166                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3167                         }
3168                         /* tell cm to disconnect, cm will queue work to thread */
3169                         nes_cm_disconn(nesqp);
3170                         break;
3171                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3172                         nesqp = *((struct nes_qp **)&context);
3173                         spin_lock_irqsave(&nesqp->lock, flags);
3174                         nesqp->hw_iwarp_state = iwarp_state;
3175                         nesqp->hw_tcp_state = tcp_state;
3176                         nesqp->last_aeq = async_event_id;
3177                         spin_unlock_irqrestore(&nesqp->lock, flags);
3178                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3179                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3180                                         nesqp->hwqp.qp_id);
3181                         if (nesqp->ibqp.event_handler) {
3182                                 ibevent.device = nesqp->ibqp.device;
3183                                 ibevent.element.qp = &nesqp->ibqp;
3184                                 ibevent.event = IB_EVENT_QP_FATAL;
3185                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3186                         }
3187                         /* tell cm to disconnect, cm will queue work to thread */
3188                         nes_cm_disconn(nesqp);
3189                         break;
3190                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3191                         nesqp = *((struct nes_qp **)&context);
3192                         spin_lock_irqsave(&nesqp->lock, flags);
3193                         nesqp->hw_iwarp_state = iwarp_state;
3194                         nesqp->hw_tcp_state = tcp_state;
3195                         nesqp->last_aeq = async_event_id;
3196                         spin_unlock_irqrestore(&nesqp->lock, flags);
3197                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3198                                         " event on QP%u \n  Q2 Data:\n",
3199                                         nesqp->hwqp.qp_id);
3200                         if (nesqp->ibqp.event_handler) {
3201                                 ibevent.device = nesqp->ibqp.device;
3202                                 ibevent.element.qp = &nesqp->ibqp;
3203                                 ibevent.event = IB_EVENT_QP_FATAL;
3204                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3205                         }
3206                         /* tell cm to disconnect, cm will queue work to thread */
3207                         nes_cm_disconn(nesqp);
3208                         break;
3209                         /* TODO: additional AEs need to be here */
3210                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3211                         nesqp = *((struct nes_qp **)&context);
3212                         spin_lock_irqsave(&nesqp->lock, flags);
3213                         nesqp->hw_iwarp_state = iwarp_state;
3214                         nesqp->hw_tcp_state = tcp_state;
3215                         nesqp->last_aeq = async_event_id;
3216                         spin_unlock_irqrestore(&nesqp->lock, flags);
3217                         if (nesqp->ibqp.event_handler) {
3218                                 ibevent.device = nesqp->ibqp.device;
3219                                 ibevent.element.qp = &nesqp->ibqp;
3220                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3221                                 nesqp->ibqp.event_handler(&ibevent,
3222                                                 nesqp->ibqp.qp_context);
3223                         }
3224                         nes_cm_disconn(nesqp);
3225                         break;
3226                 default:
3227                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3228                                         async_event_id);
3229                         break;
3230         }
3231
3232 }
3233
3234
3235 /**
3236  * nes_iwarp_ce_handler
3237  */
3238 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3239 {
3240         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3241
3242         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3243                         nescq->hw_cq.cq_number); */
3244         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3245
3246         if (nescq->ibcq.comp_handler)
3247                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3248
3249         return;
3250 }
3251
3252
3253 /**
3254  * nes_manage_apbvt()
3255  */
3256 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3257                 u32 nic_index, u32 add_port)
3258 {
3259         struct nes_device *nesdev = nesvnic->nesdev;
3260         struct nes_hw_cqp_wqe *cqp_wqe;
3261         struct nes_cqp_request *cqp_request;
3262         int ret = 0;
3263         u16 major_code;
3264
3265         /* Send manage APBVT request to CQP */
3266         cqp_request = nes_get_cqp_request(nesdev);
3267         if (cqp_request == NULL) {
3268                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3269                 return -ENOMEM;
3270         }
3271         cqp_request->waiting = 1;
3272         cqp_wqe = &cqp_request->cqp_wqe;
3273
3274         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3275                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3276                         accel_local_port, accel_local_port, nic_index);
3277
3278         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3279         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3280                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3281         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3282                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3283
3284         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3285
3286         atomic_set(&cqp_request->refcount, 2);
3287         nes_post_cqp_request(nesdev, cqp_request);
3288
3289         if (add_port == NES_MANAGE_APBVT_ADD)
3290                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3291                                 NES_EVENT_TIMEOUT);
3292         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3293                         ret, cqp_request->major_code, cqp_request->minor_code);
3294         major_code = cqp_request->major_code;
3295
3296         nes_put_cqp_request(nesdev, cqp_request);
3297
3298         if (!ret)
3299                 return -ETIME;
3300         else if (major_code)
3301                 return -EIO;
3302         else
3303                 return 0;
3304 }
3305
3306
3307 /**
3308  * nes_manage_arp_cache
3309  */
3310 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3311                 u32 ip_addr, u32 action)
3312 {
3313         struct nes_hw_cqp_wqe *cqp_wqe;
3314         struct nes_vnic *nesvnic = netdev_priv(netdev);
3315         struct nes_device *nesdev;
3316         struct nes_cqp_request *cqp_request;
3317         int arp_index;
3318
3319         nesdev = nesvnic->nesdev;
3320         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3321         if (arp_index == -1) {
3322                 return;
3323         }
3324
3325         /* update the ARP entry */
3326         cqp_request = nes_get_cqp_request(nesdev);
3327         if (cqp_request == NULL) {
3328                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3329                 return;
3330         }
3331         cqp_request->waiting = 0;
3332         cqp_wqe = &cqp_request->cqp_wqe;
3333         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3334
3335         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3336                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3337         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3338                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3339         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3340
3341         if (action == NES_ARP_ADD) {
3342                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3343                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3344                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3345                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3346                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3347                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3348         } else {
3349                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3350                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3351         }
3352
3353         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3354                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3355
3356         atomic_set(&cqp_request->refcount, 1);
3357         nes_post_cqp_request(nesdev, cqp_request);
3358 }
3359
3360
3361 /**
3362  * flush_wqes
3363  */
3364 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3365                 u32 which_wq, u32 wait_completion)
3366 {
3367         struct nes_cqp_request *cqp_request;
3368         struct nes_hw_cqp_wqe *cqp_wqe;
3369         int ret;
3370
3371         cqp_request = nes_get_cqp_request(nesdev);
3372         if (cqp_request == NULL) {
3373                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3374                 return;
3375         }
3376         if (wait_completion) {
3377                 cqp_request->waiting = 1;
3378                 atomic_set(&cqp_request->refcount, 2);
3379         } else {
3380                 cqp_request->waiting = 0;
3381         }
3382         cqp_wqe = &cqp_request->cqp_wqe;
3383         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3384
3385         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3386                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3387         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3388
3389         nes_post_cqp_request(nesdev, cqp_request);
3390
3391         if (wait_completion) {
3392                 /* Wait for CQP */
3393                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3394                                 NES_EVENT_TIMEOUT);
3395                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3396                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3397                                 ret, cqp_request->major_code, cqp_request->minor_code);
3398                 nes_put_cqp_request(nesdev, cqp_request);
3399         }
3400 }