SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static u32 crit_err_count;
  46 u32 int_mod_timer_init;
  47 u32 int_mod_cq_depth_256;
  48 u32 int_mod_cq_depth_128;
  49 u32 int_mod_cq_depth_32;
  50 u32 int_mod_cq_depth_24;
  51 u32 int_mod_cq_depth_16;
  52 u32 int_mod_cq_depth_4;
  53 u32 int_mod_cq_depth_1;
  54
  55 #include "nes_cm.h"
  56
  57 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  58 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  59 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  60                            u8 OneG_Mode);
  61 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  62 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  63 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  64 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  65                                    struct nes_hw_aeqe *aeqe);
  66 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  67 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  68
  69 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  70 static unsigned char *nes_iwarp_state_str[] = {
  71         "Non-Existant",
  72         "Idle",
  73         "RTS",
  74         "Closing",
  75         "RSVD1",
  76         "Terminate",
  77         "Error",
  78         "RSVD2",
  79 };
  80
  81 static unsigned char *nes_tcp_state_str[] = {
  82         "Non-Existant",
  83         "Closed",
  84         "Listen",
  85         "SYN Sent",
  86         "SYN Rcvd",
  87         "Established",
  88         "Close Wait",
  89         "FIN Wait 1",
  90         "Closing",
  91         "Last Ack",
  92         "FIN Wait 2",
  93         "Time Wait",
  94         "RSVD1",
  95         "RSVD2",
  96         "RSVD3",
  97         "RSVD4",
  98 };
  99 #endif
 100
 101
 102 /**
 103  * nes_nic_init_timer_defaults
 104  */
 105 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 106 {
 107         unsigned long flags;
 108         struct nes_adapter *nesadapter = nesdev->nesadapter;
 109         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 110
 111         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 112
 113         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 114         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 115         if (jumbomode) {
 116                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 117                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 118                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 119         } else {
 120                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 121                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 122                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 123         }
 124
 125         /* todo use netdev->mtu to set thresholds */
 126         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 127 }
 128
 129
 130 /**
 131  * nes_nic_init_timer
 132  */
 133 static void  nes_nic_init_timer(struct nes_device *nesdev)
 134 {
 135         unsigned long flags;
 136         struct nes_adapter *nesadapter = nesdev->nesadapter;
 137         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 138
 139         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 140
 141         if (shared_timer->timer_in_use_old == 0) {
 142                 nesdev->deepcq_count = 0;
 143                 shared_timer->timer_direction_upward = 0;
 144                 shared_timer->timer_direction_downward = 0;
 145                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 146                 shared_timer->timer_in_use_old = 0;
 147
 148         }
 149         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 150                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 151                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 152                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 153         }
 154         /* todo use netdev->mtu to set thresholds */
 155         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 156 }
 157
 158
 159 /**
 160  * nes_nic_tune_timer
 161  */
 162 static void nes_nic_tune_timer(struct nes_device *nesdev)
 163 {
 164         unsigned long flags;
 165         struct nes_adapter *nesadapter = nesdev->nesadapter;
 166         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 167         u16 cq_count = nesdev->currcq_count;
 168
 169         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 170
 171         if (shared_timer->cq_count_old <= cq_count)
 172                 shared_timer->cq_direction_downward = 0;
 173         else
 174                 shared_timer->cq_direction_downward++;
 175         shared_timer->cq_count_old = cq_count;
 176         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 177                 if (cq_count <= shared_timer->threshold_low &&
 178                     shared_timer->threshold_low > 4) {
 179                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 180                         shared_timer->cq_direction_downward=0;
 181                         nesdev->currcq_count = 0;
 182                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 183                         return;
 184                 }
 185         }
 186
 187         if (cq_count > 1) {
 188                 nesdev->deepcq_count += cq_count;
 189                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 190                         shared_timer->timer_direction_upward++;
 191                         shared_timer->timer_direction_downward = 0;
 192                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 193                         shared_timer->timer_direction_upward = 0;
 194                         shared_timer->timer_direction_downward = 0;
 195                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 196                         shared_timer->timer_direction_downward++;
 197                         shared_timer->timer_direction_upward = 0;
 198                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 199                         shared_timer->timer_in_use -= 2;
 200                         shared_timer->timer_direction_upward = 0;
 201                         shared_timer->timer_direction_downward++;
 202                 } else {
 203                         shared_timer->timer_in_use -= 4;
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward++;
 206                 }
 207
 208                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 209                         shared_timer->timer_in_use += 3;
 210                         shared_timer->timer_direction_upward = 0;
 211                         shared_timer->timer_direction_downward = 0;
 212                 }
 213                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 214                         shared_timer->timer_in_use -= 4 ;
 215                         shared_timer->timer_direction_downward = 0;
 216                         shared_timer->timer_direction_upward = 0;
 217                 }
 218         }
 219
 220         /* boundary checking */
 221         if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
 222                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
 223         else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
 224                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
 225         }
 226
 227         nesdev->currcq_count = 0;
 228
 229         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 230 }
 231
 232
 233 /**
 234  * nes_init_adapter - initialize adapter
 235  */
 236 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 237         struct nes_adapter *nesadapter = NULL;
 238         unsigned long num_pds;
 239         u32 u32temp;
 240         u32 port_count;
 241         u16 max_rq_wrs;
 242         u16 max_sq_wrs;
 243         u32 max_mr;
 244         u32 max_256pbl;
 245         u32 max_4kpbl;
 246         u32 max_qp;
 247         u32 max_irrq;
 248         u32 max_cq;
 249         u32 hte_index_mask;
 250         u32 adapter_size;
 251         u32 arp_table_size;
 252         u16 vendor_id;
 253         u8  OneG_Mode;
 254         u8  func_index;
 255
 256         /* search the list of existing adapters */
 257         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 258                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 259                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 260                                 nesdev->pcidev->devfn,
 261                                 PCI_SLOT(nesadapter->devfn),
 262                                 nesadapter->bus_number,
 263                                 PCI_SLOT(nesdev->pcidev->devfn),
 264                                 nesdev->pcidev->bus->number );
 265                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 266                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 267                         nesadapter->ref_count++;
 268                         return nesadapter;
 269                 }
 270         }
 271
 272         /* no adapter found */
 273         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 274         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 275                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 276                                 hw_rev);
 277                 return NULL;
 278         }
 279
 280         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 281                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 282                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 283                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 284                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 285
 286         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 287
 288
 289         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 290                 return NULL;
 291         if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
 292                 return NULL;
 293         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 294
 295         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 296         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 297
 298         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 299         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 300                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 301                                 max_qp, u32temp);
 302                 max_qp = (u32)1 << (u32temp & 0x001f);
 303         }
 304
 305         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 306         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 307                         max_qp, hte_index_mask);
 308
 309         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 310
 311         max_irrq = 1 << (u32temp & 0x001f);
 312
 313         if (max_qp > max_irrq) {
 314                 max_qp = max_irrq;
 315                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 316                                 max_qp);
 317         }
 318
 319         /* there should be no reason to allocate more pds than qps */
 320         if (num_pds > max_qp)
 321                 num_pds = max_qp;
 322
 323         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 324         max_mr = (u32)8192 << (u32temp & 0x7);
 325
 326         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 327         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 328         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 329         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 330
 331         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 332         arp_table_size = 1 << u32temp;
 333
 334         adapter_size = (sizeof(struct nes_adapter) +
 335                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 336         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 337         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 340         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 341         adapter_size += sizeof(struct nes_qp **) * max_qp;
 342
 343         /* allocate a new adapter struct */
 344         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 345         if (nesadapter == NULL) {
 346                 return NULL;
 347         }
 348
 349         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 350                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 351
 352         /* populate the new nesadapter */
 353         nesadapter->devfn = nesdev->pcidev->devfn;
 354         nesadapter->bus_number = nesdev->pcidev->bus->number;
 355         nesadapter->ref_count = 1;
 356         nesadapter->timer_int_req = 0xffff0000;
 357         nesadapter->OneG_Mode = OneG_Mode;
 358         nesadapter->doorbell_start = nesdev->doorbell_region;
 359
 360         /* nesadapter->tick_delta = clk_divisor; */
 361         nesadapter->hw_rev = hw_rev;
 362         nesadapter->port_count = port_count;
 363
 364         nesadapter->max_qp = max_qp;
 365         nesadapter->hte_index_mask = hte_index_mask;
 366         nesadapter->max_irrq = max_irrq;
 367         nesadapter->max_mr = max_mr;
 368         nesadapter->max_256pbl = max_256pbl - 1;
 369         nesadapter->max_4kpbl = max_4kpbl - 1;
 370         nesadapter->max_cq = max_cq;
 371         nesadapter->free_256pbl = max_256pbl - 1;
 372         nesadapter->free_4kpbl = max_4kpbl - 1;
 373         nesadapter->max_pd = num_pds;
 374         nesadapter->arp_table_size = arp_table_size;
 375
 376         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 377         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 378                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 379                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 380                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 381         } else {
 382                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 383                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 384                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 385                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 386         }
 387         /* Setup and enable the periodic timer */
 388         if (nesadapter->et_rx_coalesce_usecs_irq)
 389                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 390                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 391         else
 392                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 393
 394         nesadapter->base_pd = 1;
 395
 396         nesadapter->device_cap_flags =
 397                 IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
 398
 399         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 400                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 401         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 402         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 403         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 404         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 405         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 406
 407
 408         /* mark the usual suspect QPs and CQs as in use */
 409         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 410                 set_bit(u32temp, nesadapter->allocated_qps);
 411                 set_bit(u32temp, nesadapter->allocated_cqs);
 412         }
 413
 414         for (u32temp = 0; u32temp < 20; u32temp++)
 415                 set_bit(u32temp, nesadapter->allocated_pds);
 416         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 417
 418         max_rq_wrs = ((u32temp >> 8) & 3);
 419         switch (max_rq_wrs) {
 420                 case 0:
 421                         max_rq_wrs = 4;
 422                         break;
 423                 case 1:
 424                         max_rq_wrs = 16;
 425                         break;
 426                 case 2:
 427                         max_rq_wrs = 32;
 428                         break;
 429                 case 3:
 430                         max_rq_wrs = 512;
 431                         break;
 432         }
 433
 434         max_sq_wrs = (u32temp & 3);
 435         switch (max_sq_wrs) {
 436                 case 0:
 437                         max_sq_wrs = 4;
 438                         break;
 439                 case 1:
 440                         max_sq_wrs = 16;
 441                         break;
 442                 case 2:
 443                         max_sq_wrs = 32;
 444                         break;
 445                 case 3:
 446                         max_sq_wrs = 512;
 447                         break;
 448         }
 449         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 450         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 451
 452         nesadapter->max_sge = 4;
 453         nesadapter->max_cqe = 32767;
 454
 455         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 456                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 457                 kfree(nesadapter);
 458                 return NULL;
 459         }
 460
 461         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 462         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 463                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 464
 465         /* setup port configuration */
 466         if (nesadapter->port_count == 1) {
 467                 u32temp = 0x00000000;
 468                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 469                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 470                 else
 471                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 472         } else {
 473                 if (nesadapter->port_count == 2)
 474                         u32temp = 0x00000044;
 475                 else
 476                         u32temp = 0x000000e4;
 477                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 478         }
 479
 480         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
 481         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 482                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 483
 484         spin_lock_init(&nesadapter->resource_lock);
 485         spin_lock_init(&nesadapter->phy_lock);
 486         spin_lock_init(&nesadapter->pbl_lock);
 487         spin_lock_init(&nesadapter->periodic_timer_lock);
 488
 489         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 490         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 491         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 492         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 493
 494         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 495                 u32 pcs_control_status0, pcs_control_status1;
 496                 u32 reset_value;
 497                 u32 i = 0;
 498                 u32 int_cnt = 0;
 499                 u32 ext_cnt = 0;
 500                 unsigned long flags;
 501                 u32 j = 0;
 502
 503                 pcs_control_status0 = nes_read_indexed(nesdev,
 504                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 505                 pcs_control_status1 = nes_read_indexed(nesdev,
 506                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 507
 508                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 509                         pcs_control_status0 = nes_read_indexed(nesdev,
 510                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 511                         pcs_control_status1 = nes_read_indexed(nesdev,
 512                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 513                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 514                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 515                                 int_cnt++;
 516                         msleep(1);
 517                 }
 518                 if (int_cnt > 1) {
 519                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 520                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 521                         mh_detected++;
 522                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 523                         reset_value |= 0x0000003d;
 524                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 525
 526                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 527                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 528                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 529
 530                         pcs_control_status0 = nes_read_indexed(nesdev,
 531                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 532                         pcs_control_status1 = nes_read_indexed(nesdev,
 533                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 534
 535                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 536                                 pcs_control_status0 = nes_read_indexed(nesdev,
 537                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 538                                 pcs_control_status1 = nes_read_indexed(nesdev,
 539                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 540                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 541                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 542                                         if (++ext_cnt > int_cnt) {
 543                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 544                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 545                                                                 0x0000F0C8);
 546                                                 mh_detected++;
 547                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 548                                                 reset_value |= 0x0000003d;
 549                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 550
 551                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 552                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 553                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 554                                                 break;
 555                                         }
 556                                 }
 557                                 msleep(1);
 558                         }
 559                 }
 560         }
 561
 562         if (nesadapter->hw_rev == NE020_REV) {
 563                 init_timer(&nesadapter->mh_timer);
 564                 nesadapter->mh_timer.function = nes_mh_fix;
 565                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 566                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 567                 add_timer(&nesadapter->mh_timer);
 568         } else {
 569                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 570         }
 571
 572         init_timer(&nesadapter->lc_timer);
 573         nesadapter->lc_timer.function = nes_clc;
 574         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 575         nesadapter->lc_timer.data = (unsigned long)nesdev;
 576         add_timer(&nesadapter->lc_timer);
 577
 578         list_add_tail(&nesadapter->list, &nes_adapter_list);
 579
 580         for (func_index = 0; func_index < 8; func_index++) {
 581                 pci_bus_read_config_word(nesdev->pcidev->bus,
 582                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 583                                         func_index), 0, &vendor_id);
 584                 if (vendor_id == 0xffff)
 585                         break;
 586         }
 587         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 588                 func_index, pci_name(nesdev->pcidev));
 589         nesadapter->adapter_fcn_count = func_index;
 590
 591         return nesadapter;
 592 }
 593
 594
 595 /**
 596  * nes_reset_adapter_ne020
 597  */
 598 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 599 {
 600         u32 port_count;
 601         u32 u32temp;
 602         u32 i;
 603
 604         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 605         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 606         /* TODO: assuming that both SERDES are set the same for now */
 607         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 608         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 609                         u32temp, port_count);
 610         if (*OneG_Mode)
 611                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 612         u32temp &= 0xff00ffc0;
 613         switch (port_count) {
 614                 case 1:
 615                         u32temp |= 0x00ee0000;
 616                         break;
 617                 case 2:
 618                         u32temp |= 0x00cc0000;
 619                         break;
 620                 case 4:
 621                         u32temp |= 0x00000000;
 622                         break;
 623                 default:
 624                         return 0;
 625                         break;
 626         }
 627
 628         /* check and do full reset if needed */
 629         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 630                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 631                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 632
 633                 i = 0;
 634                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 635                         mdelay(1);
 636                 if (i >= 10000) {
 637                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 638                         return 0;
 639                 }
 640
 641                 i = 0;
 642                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 643                         mdelay(1);
 644                 if (i >= 10000) {
 645                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 646                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 647                         return 0;
 648                 }
 649         }
 650
 651         /* port reset */
 652         switch (port_count) {
 653                 case 1:
 654                         u32temp |= 0x00ee0010;
 655                         break;
 656                 case 2:
 657                         u32temp |= 0x00cc0030;
 658                         break;
 659                 case 4:
 660                         u32temp |= 0x00000030;
 661                         break;
 662         }
 663
 664         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 665         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 666
 667         i = 0;
 668         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 669                 mdelay(1);
 670         if (i >= 10000) {
 671                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 672                 return 0;
 673         }
 674
 675         /* serdes 0 */
 676         i = 0;
 677         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 678                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 679                 mdelay(1);
 680         if (i >= 5000) {
 681                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 682                 return 0;
 683         }
 684
 685         /* serdes 1 */
 686         if (port_count > 1) {
 687                 i = 0;
 688                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 689                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 690                         mdelay(1);
 691                 if (i >= 5000) {
 692                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 693                         return 0;
 694                 }
 695         }
 696
 697         return port_count;
 698 }
 699
 700
 701 /**
 702  * nes_init_serdes
 703  */
 704 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 705                            u8 OneG_Mode)
 706 {
 707         int i;
 708         u32 u32temp;
 709
 710         if (hw_rev != NE020_REV) {
 711                 /* init serdes 0 */
 712
 713                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 714                 if (!OneG_Mode)
 715                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 716                 if (port_count > 1) {
 717                         /* init serdes 1 */
 718                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 719                         if (!OneG_Mode)
 720                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 721                         }
 722         } else {
 723                 /* init serdes 0 */
 724                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 725                 i = 0;
 726                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 727                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 728                         mdelay(1);
 729                 if (i >= 5000) {
 730                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 731                         return 1;
 732                 }
 733                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 734                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 735                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 736                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 737                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 738                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 739                 if (OneG_Mode)
 740                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 741                 else
 742                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 743
 744                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 745                 if (port_count > 1) {
 746                         /* init serdes 1 */
 747                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 748                         i = 0;
 749                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 750                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 751                                 mdelay(1);
 752                         if (i >= 5000) {
 753                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 754                                 /* return 1; */
 755                         }
 756                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 757                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 762                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 764                 }
 765         }
 766         return 0;
 767 }
 768
 769
 770 /**
 771  * nes_init_csr_ne020
 772  * Initialize registers for ne020 hardware
 773  */
 774 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 775 {
 776         u32 u32temp;
 777
 778         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 779
 780         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 781         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 782         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 783         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 784         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 785         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 786         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 787         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 788
 789         /* TODO: move these MAC register settings to NIC bringup */
 790         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 791         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 792         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 793         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 794         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 795         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 796         if (port_count > 1) {
 797                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 798                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 799                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 800                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 801                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 802                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 803                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 804         }
 805         if (port_count > 2) {
 806                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 807                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 808                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 809                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 810                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 811                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 812                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 813
 814                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 815                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 816                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 817                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 818                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 819                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 820                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 821         }
 822
 823         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 824         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 825         nes_write_indexed(nesdev, 0x00005004, 0x00020001);
 826         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 827         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 828         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 829         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 830         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 831
 832         /* TODO: move this to code, get from EEPROM */
 833         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 834         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 835         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 836                                                                                                                 //
 837         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 838         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 839
 840         if (hw_rev != NE020_REV) {
 841                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 842                 u32temp |= 0x80000000;
 843                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 844                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 845                 u32temp &= 0x7fffffff;
 846                 u32temp |= 0x7fff0010;
 847                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 848         }
 849 }
 850
 851
 852 /**
 853  * nes_destroy_adapter - destroy the adapter structure
 854  */
 855 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 856 {
 857         struct nes_adapter *tmp_adapter;
 858
 859         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 860                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 861                                 tmp_adapter);
 862         }
 863
 864         nesadapter->ref_count--;
 865         if (!nesadapter->ref_count) {
 866                 if (nesadapter->hw_rev == NE020_REV) {
 867                         del_timer(&nesadapter->mh_timer);
 868                 }
 869                 del_timer(&nesadapter->lc_timer);
 870
 871                 list_del(&nesadapter->list);
 872                 kfree(nesadapter);
 873         }
 874 }
 875
 876
 877 /**
 878  * nes_init_cqp
 879  */
 880 int nes_init_cqp(struct nes_device *nesdev)
 881 {
 882         struct nes_adapter *nesadapter = nesdev->nesadapter;
 883         struct nes_hw_cqp_qp_context *cqp_qp_context;
 884         struct nes_hw_cqp_wqe *cqp_wqe;
 885         struct nes_hw_ceq *ceq;
 886         struct nes_hw_ceq *nic_ceq;
 887         struct nes_hw_aeq *aeq;
 888         void *vmem;
 889         dma_addr_t pmem;
 890         u32 count=0;
 891         u32 cqp_head;
 892         u64 u64temp;
 893         u32 u32temp;
 894
 895         /* allocate CQP memory */
 896         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 897         /* SQ is 512 byte aligned, others are 256 byte aligned */
 898         nesdev->cqp_mem_size = 512 +
 899                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 900                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 901                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 902                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 903                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 904                         sizeof(struct nes_hw_cqp_qp_context);
 905
 906         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 907                         &nesdev->cqp_pbase);
 908         if (!nesdev->cqp_vbase) {
 909                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 910                 return -ENOMEM;
 911         }
 912         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 913
 914         /* Allocate a twice the number of CQP requests as the SQ size */
 915         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 916                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 917         if (nesdev->nes_cqp_requests == NULL) {
 918                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 919                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 920                                 nesdev->cqp.sq_pbase);
 921                 return -ENOMEM;
 922         }
 923
 924         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 925                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 926
 927         spin_lock_init(&nesdev->cqp.lock);
 928         init_waitqueue_head(&nesdev->cqp.waitq);
 929
 930         /* Setup Various Structures */
 931         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 932                         ~(unsigned long)(512 - 1));
 933         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 934                         ~(unsigned long long)(512 - 1));
 935
 936         nesdev->cqp.sq_vbase = vmem;
 937         nesdev->cqp.sq_pbase = pmem;
 938         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 939         nesdev->cqp.sq_head = 0;
 940         nesdev->cqp.sq_tail = 0;
 941         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 942
 943         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 944         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 945
 946         nesdev->ccq.cq_vbase = vmem;
 947         nesdev->ccq.cq_pbase = pmem;
 948         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 949         nesdev->ccq.cq_head = 0;
 950         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 951         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 952
 953         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 954         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 955
 956         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
 957         ceq = &nesadapter->ceq[nesdev->ceq_index];
 958         ceq->ceq_vbase = vmem;
 959         ceq->ceq_pbase = pmem;
 960         ceq->ceq_size = NES_CCEQ_SIZE;
 961         ceq->ceq_head = 0;
 962
 963         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 964         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 965
 966         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
 967         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
 968         nic_ceq->ceq_vbase = vmem;
 969         nic_ceq->ceq_pbase = pmem;
 970         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
 971         nic_ceq->ceq_head = 0;
 972
 973         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 974         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 975
 976         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
 977         aeq->aeq_vbase = vmem;
 978         aeq->aeq_pbase = pmem;
 979         aeq->aeq_size = nesadapter->max_qp;
 980         aeq->aeq_head = 0;
 981
 982         /* Setup QP Context */
 983         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 984         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 985
 986         cqp_qp_context = vmem;
 987         cqp_qp_context->context_words[0] =
 988                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
 989         cqp_qp_context->context_words[1] = 0;
 990         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
 991         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
 992
 993
 994         /* Write the address to Create CQP */
 995         if ((sizeof(dma_addr_t) > 4)) {
 996                 nes_write_indexed(nesdev,
 997                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 998                                 ((u64)pmem) >> 32);
 999         } else {
1000                 nes_write_indexed(nesdev,
1001                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1002         }
1003         nes_write_indexed(nesdev,
1004                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1005                         (u32)pmem);
1006
1007         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1008         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1009
1010         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1011                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1012                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1013         }
1014
1015         /* Write Create CCQ WQE */
1016         cqp_head = nesdev->cqp.sq_head++;
1017         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1018         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1019         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1020                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1021                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1022         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1023                             (nesdev->ccq.cq_number |
1024                              ((u32)nesdev->ceq_index << 16)));
1025         u64temp = (u64)nesdev->ccq.cq_pbase;
1026         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1027         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1028         u64temp = (unsigned long)&nesdev->ccq;
1029         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1030                         cpu_to_le32((u32)(u64temp >> 1));
1031         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1032                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1033         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1034
1035         /* Write Create CEQ WQE */
1036         cqp_head = nesdev->cqp.sq_head++;
1037         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1038         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1039         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1040                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1041         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1042         u64temp = (u64)ceq->ceq_pbase;
1043         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1044
1045         /* Write Create AEQ WQE */
1046         cqp_head = nesdev->cqp.sq_head++;
1047         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1048         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1049         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1050                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1051         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1052         u64temp = (u64)aeq->aeq_pbase;
1053         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1054
1055         /* Write Create NIC CEQ WQE */
1056         cqp_head = nesdev->cqp.sq_head++;
1057         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1058         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1059         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1060                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1061         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1062         u64temp = (u64)nic_ceq->ceq_pbase;
1063         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1064
1065         /* Poll until CCQP done */
1066         count = 0;
1067         do {
1068                 if (count++ > 1000) {
1069                         printk(KERN_ERR PFX "Error creating CQP\n");
1070                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1071                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1072                         return -1;
1073                 }
1074                 udelay(10);
1075         } while (!(nes_read_indexed(nesdev,
1076                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1077
1078         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1079                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1080
1081         u32temp = 0x04800000;
1082         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1083
1084         /* wait for the CCQ, CEQ, and AEQ to get created */
1085         count = 0;
1086         do {
1087                 if (count++ > 1000) {
1088                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1089                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1090                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1091                         return -1;
1092                 }
1093                 udelay(10);
1094         } while (((nes_read_indexed(nesdev,
1095                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1096
1097         /* dump the QP status value */
1098         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1099                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1100
1101         nesdev->cqp.sq_tail++;
1102
1103         return 0;
1104 }
1105
1106
1107 /**
1108  * nes_destroy_cqp
1109  */
1110 int nes_destroy_cqp(struct nes_device *nesdev)
1111 {
1112         struct nes_hw_cqp_wqe *cqp_wqe;
1113         u32 count = 0;
1114         u32 cqp_head;
1115         unsigned long flags;
1116
1117         do {
1118                 if (count++ > 1000)
1119                         break;
1120                 udelay(10);
1121         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1122
1123         /* Reset CCQ */
1124         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1125                         nesdev->ccq.cq_number);
1126
1127         /* Disable device interrupts */
1128         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1129
1130         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1131
1132         /* Destroy the AEQ */
1133         cqp_head = nesdev->cqp.sq_head++;
1134         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1135         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1136         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1137                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1138         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1139
1140         /* Destroy the NIC CEQ */
1141         cqp_head = nesdev->cqp.sq_head++;
1142         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1143         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1144         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1145                         ((u32)nesdev->nic_ceq_index << 8));
1146
1147         /* Destroy the CEQ */
1148         cqp_head = nesdev->cqp.sq_head++;
1149         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1150         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1151         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1152                         (nesdev->ceq_index << 8));
1153
1154         /* Destroy the CCQ */
1155         cqp_head = nesdev->cqp.sq_head++;
1156         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1157         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1158         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1159         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1160                         ((u32)nesdev->ceq_index << 16));
1161
1162         /* Destroy CQP */
1163         cqp_head = nesdev->cqp.sq_head++;
1164         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1165         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1166         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1167                         NES_CQP_QP_TYPE_CQP);
1168         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1169
1170         barrier();
1171         /* Ring doorbell (5 WQEs) */
1172         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1173
1174         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1175
1176         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1177         count = 0;
1178         do {
1179                 if (count++ > 1000) {
1180                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1181                                         PCI_FUNC(nesdev->pcidev->devfn));
1182                         break;
1183                 }
1184                 udelay(10);
1185         } while (((nes_read_indexed(nesdev,
1186                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1187
1188         /* dump the QP status value */
1189         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1190                         PCI_FUNC(nesdev->pcidev->devfn),
1191                         nes_read_indexed(nesdev,
1192                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1193
1194         kfree(nesdev->nes_cqp_requests);
1195
1196         /* Free the control structures */
1197         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1198                         nesdev->cqp.sq_pbase);
1199
1200         return 0;
1201 }
1202
1203
1204 /**
1205  * nes_init_phy
1206  */
1207 int nes_init_phy(struct nes_device *nesdev)
1208 {
1209         struct nes_adapter *nesadapter = nesdev->nesadapter;
1210         u32 counter = 0;
1211         u32 sds_common_control0;
1212         u32 mac_index = nesdev->mac_index;
1213         u32 tx_config = 0;
1214         u16 phy_data;
1215         u32 temp_phy_data = 0;
1216         u32 temp_phy_data2 = 0;
1217         u32 i = 0;
1218
1219         if ((nesadapter->OneG_Mode) &&
1220             (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
1221                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1222                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1223                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1224                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1225                         tx_config |= 0x04;
1226                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1227                 }
1228
1229                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1230                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1231                                 nesadapter->phy_index[mac_index], phy_data);
1232                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index],  0xb000);
1233
1234                 /* Reset the PHY */
1235                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1236                 udelay(100);
1237                 counter = 0;
1238                 do {
1239                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1240                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1241                         if (counter++ > 100) break;
1242                 } while (phy_data & 0x8000);
1243
1244                 /* Setting no phy loopback */
1245                 phy_data &= 0xbfff;
1246                 phy_data |= 0x1140;
1247                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1248                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1249                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1250
1251                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1252                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1253
1254                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1255                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1256
1257                 /* Setting the interrupt mask */
1258                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1259                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1260                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1261
1262                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1263                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1264
1265                 /* turning on flow control */
1266                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1267                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1268                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1269                                 (phy_data & ~(0x03E0)) | 0xc00);
1270                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1271                                 phy_data | 0xc00); */
1272                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1273                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1274
1275                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1276                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1277                 /* Clear Half duplex */
1278                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1279                                 phy_data & ~(0x0100));
1280                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1281                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1282
1283                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1284                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1285         } else {
1286                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) ||
1287                     (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1288                         /* setup 10G MDIO operation */
1289                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1290                         tx_config |= 0x14;
1291                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1292                 }
1293                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1294                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1295
1296                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1297                         mdelay(10);
1298                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1299                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1300
1301                         /*
1302                          * if firmware is already running (like from a
1303                          * driver un-load/load, don't do anything.
1304                          */
1305                         if (temp_phy_data == temp_phy_data2) {
1306                                 /* configure QT2505 AMCC PHY */
1307                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0x0000, 0x8000);
1308                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0000);
1309                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc302, 0x0044);
1310                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc318, 0x0052);
1311                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc319, 0x0008);
1312                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc31a, 0x0098);
1313                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0026, 0x0E00);
1314                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0027, 0x0000);
1315                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0028, 0xA528);
1316
1317                                 /*
1318                                  * remove micro from reset; chip boots from ROM,
1319                                  * uploads EEPROM f/w image, uC executes f/w
1320                                  */
1321                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0002);
1322
1323                                 /*
1324                                  * wait for heart beat to start to
1325                                  * know loading is done
1326                                  */
1327                                 counter = 0;
1328                                 do {
1329                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1330                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1331                                         if (counter++ > 1000) {
1332                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from heartbeat check <this is bad!!!> \n");
1333                                                 break;
1334                                         }
1335                                         mdelay(100);
1336                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1337                                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1338                                 } while ((temp_phy_data2 == temp_phy_data));
1339
1340                                 /*
1341                                  * wait for tracking to start to know
1342                                  * f/w is good to go
1343                                  */
1344                                 counter = 0;
1345                                 do {
1346                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7fd);
1347                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1348                                         if (counter++ > 1000) {
1349                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from status check <this is bad!!!> \n");
1350                                                 break;
1351                                         }
1352                                         mdelay(1000);
1353                                         /*
1354                                          * nes_debug(NES_DBG_PHY, "AMCC PHY- phy_status not ready yet = 0x%02X\n",
1355                                          *                      temp_phy_data);
1356                                          */
1357                                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1358
1359                                 /* set LOS Control invert RXLOSB_I_PADINV */
1360                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd003, 0x0000);
1361                                 /* set LOS Control to mask of RXLOSB_I */
1362                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc314, 0x0042);
1363                                 /* set LED1 to input mode (LED1 and LED2 share same LED) */
1364                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd006, 0x0007);
1365                                 /* set LED2 to RX link_status and activity */
1366                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd007, 0x000A);
1367                                 /* set LED3 to RX link_status */
1368                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd008, 0x0009);
1369
1370                                 /*
1371                                  * reset the res-calibration on t2
1372                                  * serdes; ensures it is stable after
1373                                  * the amcc phy is stable
1374                                  */
1375
1376                                 sds_common_control0 = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
1377                                 sds_common_control0 |= 0x1;
1378                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1379
1380                                 /* release the res-calibration reset */
1381                                 sds_common_control0 &= 0xfffffffe;
1382                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1383
1384                                 i = 0;
1385                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1386                                                 && (i++ < 5000)) {
1387                                         /* mdelay(1); */
1388                                 }
1389
1390                                 /*
1391                                  * wait for link train done before moving on,
1392                                  * or will get an interupt storm
1393                                  */
1394                                 counter = 0;
1395                                 do {
1396                                         temp_phy_data = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
1397                                                                 (0x200 * (nesdev->mac_index & 1)));
1398                                         if (counter++ > 1000) {
1399                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from link train wait <this is bad, link didnt train!!!>\n");
1400                                                 break;
1401                                         }
1402                                         mdelay(1);
1403                                 } while (((temp_phy_data & 0x0f1f0000) != 0x0f0f0000));
1404                         }
1405                 }
1406         }
1407         return 0;
1408 }
1409
1410
1411 /**
1412  * nes_replenish_nic_rq
1413  */
1414 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1415 {
1416         unsigned long flags;
1417         dma_addr_t bus_address;
1418         struct sk_buff *skb;
1419         struct nes_hw_nic_rq_wqe *nic_rqe;
1420         struct nes_hw_nic *nesnic;
1421         struct nes_device *nesdev;
1422         u32 rx_wqes_posted = 0;
1423
1424         nesnic = &nesvnic->nic;
1425         nesdev = nesvnic->nesdev;
1426         spin_lock_irqsave(&nesnic->rq_lock, flags);
1427         if (nesnic->replenishing_rq !=0) {
1428                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1429                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1430                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1431                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1432                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1433                         add_timer(&nesvnic->rq_wqes_timer);
1434                 } else
1435                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1436                 return;
1437         }
1438         nesnic->replenishing_rq = 1;
1439         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1440         do {
1441                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1442                 if (skb) {
1443                         skb->dev = nesvnic->netdev;
1444
1445                         bus_address = pci_map_single(nesdev->pcidev,
1446                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1447
1448                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1449                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1450                                         cpu_to_le32(nesvnic->max_frame_size);
1451                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1452                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1453                                         cpu_to_le32((u32)bus_address);
1454                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1455                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1456                         nesnic->rx_skb[nesnic->rq_head] = skb;
1457                         nesnic->rq_head++;
1458                         nesnic->rq_head &= nesnic->rq_size - 1;
1459                         atomic_dec(&nesvnic->rx_skbs_needed);
1460                         barrier();
1461                         if (++rx_wqes_posted == 255) {
1462                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1463                                 rx_wqes_posted = 0;
1464                         }
1465                 } else {
1466                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1467                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1468                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1469                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1470                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1471                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1472                                 add_timer(&nesvnic->rq_wqes_timer);
1473                         } else
1474                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1475                         break;
1476                 }
1477         } while (atomic_read(&nesvnic->rx_skbs_needed));
1478         barrier();
1479         if (rx_wqes_posted)
1480                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1481         nesnic->replenishing_rq = 0;
1482 }
1483
1484
1485 /**
1486  * nes_rq_wqes_timeout
1487  */
1488 static void nes_rq_wqes_timeout(unsigned long parm)
1489 {
1490         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1491         printk("%s: Timer fired.\n", __func__);
1492         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1493         if (atomic_read(&nesvnic->rx_skbs_needed))
1494                 nes_replenish_nic_rq(nesvnic);
1495 }
1496
1497
1498 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1499                                void **tcph, u64 *hdr_flags, void *priv)
1500 {
1501         unsigned int ip_len;
1502         struct iphdr *iph;
1503         skb_reset_network_header(skb);
1504         iph = ip_hdr(skb);
1505         if (iph->protocol != IPPROTO_TCP)
1506                 return -1;
1507         ip_len = ip_hdrlen(skb);
1508         skb_set_transport_header(skb, ip_len);
1509         *tcph = tcp_hdr(skb);
1510
1511         *hdr_flags = LRO_IPV4 | LRO_TCP;
1512         *iphdr = iph;
1513         return 0;
1514 }
1515
1516
1517 /**
1518  * nes_init_nic_qp
1519  */
1520 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1521 {
1522         struct nes_hw_cqp_wqe *cqp_wqe;
1523         struct nes_hw_nic_sq_wqe *nic_sqe;
1524         struct nes_hw_nic_qp_context *nic_context;
1525         struct sk_buff *skb;
1526         struct nes_hw_nic_rq_wqe *nic_rqe;
1527         struct nes_vnic *nesvnic = netdev_priv(netdev);
1528         unsigned long flags;
1529         void *vmem;
1530         dma_addr_t pmem;
1531         u64 u64temp;
1532         int ret;
1533         u32 cqp_head;
1534         u32 counter;
1535         u32 wqe_count;
1536         u8 jumbomode=0;
1537
1538         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1539         nesvnic->nic_mem_size = 256 +
1540                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1541                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1542                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1543                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1544                         sizeof(struct nes_hw_nic_qp_context);
1545
1546         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1547                         &nesvnic->nic_pbase);
1548         if (!nesvnic->nic_vbase) {
1549                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1550                 return -ENOMEM;
1551         }
1552         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1553         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1554                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1555
1556         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1557                         ~(unsigned long)(256 - 1));
1558         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1559                         ~(unsigned long long)(256 - 1));
1560
1561         /* Setup the first Fragment buffers */
1562         nesvnic->nic.first_frag_vbase = vmem;
1563
1564         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1565                 nesvnic->nic.frag_paddr[counter] = pmem;
1566                 pmem += sizeof(struct nes_first_frag);
1567         }
1568
1569         /* setup the SQ */
1570         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1571
1572         nesvnic->nic.sq_vbase = (void *)vmem;
1573         nesvnic->nic.sq_pbase = pmem;
1574         nesvnic->nic.sq_head = 0;
1575         nesvnic->nic.sq_tail = 0;
1576         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1577         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1578                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1579                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1580                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1581                                 NES_NIC_SQ_WQE_COMPLETION);
1582                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1583                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1584                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1585                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1586                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1587                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1588         }
1589
1590         nesvnic->get_cqp_request = nes_get_cqp_request;
1591         nesvnic->post_cqp_request = nes_post_cqp_request;
1592         nesvnic->mcrq_mcast_filter = NULL;
1593
1594         spin_lock_init(&nesvnic->nic.sq_lock);
1595         spin_lock_init(&nesvnic->nic.rq_lock);
1596
1597         /* setup the RQ */
1598         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1599         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1600
1601
1602         nesvnic->nic.rq_vbase = vmem;
1603         nesvnic->nic.rq_pbase = pmem;
1604         nesvnic->nic.rq_head = 0;
1605         nesvnic->nic.rq_tail = 0;
1606         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1607
1608         /* setup the CQ */
1609         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1610         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1611
1612         if (nesdev->nesadapter->netdev_count > 2)
1613                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1614         else
1615                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1616
1617         nesvnic->nic_cq.cq_vbase = vmem;
1618         nesvnic->nic_cq.cq_pbase = pmem;
1619         nesvnic->nic_cq.cq_head = 0;
1620         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1621
1622         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1623
1624         /* Send CreateCQ request to CQP */
1625         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1626         cqp_head = nesdev->cqp.sq_head;
1627
1628         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1629         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1630
1631         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1632                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1633                         ((u32)nesvnic->nic_cq.cq_size << 16));
1634         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1635                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1636         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1637         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1638         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1639         u64temp = (unsigned long)&nesvnic->nic_cq;
1640         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1641         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1642                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1643         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1644         if (++cqp_head >= nesdev->cqp.sq_size)
1645                 cqp_head = 0;
1646         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1647         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1648
1649         /* Send CreateQP request to CQP */
1650         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1651         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1652                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1653                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1654         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1655                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1656                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1657         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1658                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1659         }
1660
1661         u64temp = (u64)nesvnic->nic.sq_pbase;
1662         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1663         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1664         u64temp = (u64)nesvnic->nic.rq_pbase;
1665         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1666         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1667
1668         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1669                         NES_CQP_QP_TYPE_NIC);
1670         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1671         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1672                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1673         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1674
1675         if (++cqp_head >= nesdev->cqp.sq_size)
1676                 cqp_head = 0;
1677         nesdev->cqp.sq_head = cqp_head;
1678
1679         barrier();
1680
1681         /* Ring doorbell (2 WQEs) */
1682         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1683
1684         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1685         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1686                         nesvnic->nic.qp_id);
1687
1688         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1689                         NES_EVENT_TIMEOUT);
1690         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1691                         nesvnic->nic.qp_id, ret);
1692         if (!ret) {
1693                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1694                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1695                                 nesvnic->nic_pbase);
1696                 return -EIO;
1697         }
1698
1699         /* Populate the RQ */
1700         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1701                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1702                 if (!skb) {
1703                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1704
1705                         nes_destroy_nic_qp(nesvnic);
1706                         return -ENOMEM;
1707                 }
1708
1709                 skb->dev = netdev;
1710
1711                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1712                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1713
1714                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1715                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1716                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1717                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1718                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1719                 nesvnic->nic.rx_skb[counter] = skb;
1720         }
1721
1722         wqe_count = NES_NIC_WQ_SIZE - 1;
1723         nesvnic->nic.rq_head = wqe_count;
1724         barrier();
1725         do {
1726                 counter = min(wqe_count, ((u32)255));
1727                 wqe_count -= counter;
1728                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1729         } while (wqe_count);
1730         init_timer(&nesvnic->rq_wqes_timer);
1731         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1732         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1733         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1734         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1735         {
1736                 nes_nic_init_timer(nesdev);
1737                 if (netdev->mtu > 1500)
1738                         jumbomode = 1;
1739                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1740         }
1741         nesvnic->lro_mgr.max_aggr = NES_LRO_MAX_AGGR;
1742         nesvnic->lro_mgr.max_desc = NES_MAX_LRO_DESCRIPTORS;
1743         nesvnic->lro_mgr.lro_arr = nesvnic->lro_desc;
1744         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1745         nesvnic->lro_mgr.features = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1746         nesvnic->lro_mgr.dev = netdev;
1747         nesvnic->lro_mgr.ip_summed = CHECKSUM_UNNECESSARY;
1748         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1749         return 0;
1750 }
1751
1752
1753 /**
1754  * nes_destroy_nic_qp
1755  */
1756 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1757 {
1758         struct nes_device *nesdev = nesvnic->nesdev;
1759         struct nes_hw_cqp_wqe *cqp_wqe;
1760         struct nes_hw_nic_rq_wqe *nic_rqe;
1761         u64 wqe_frag;
1762         u32 cqp_head;
1763         unsigned long flags;
1764         int ret;
1765
1766         /* Free remaining NIC receive buffers */
1767         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1768                 nic_rqe = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1769                 wqe_frag = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1770                 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1771                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1772                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1773                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1774                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1775         }
1776
1777         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1778
1779         /* Destroy NIC QP */
1780         cqp_head = nesdev->cqp.sq_head;
1781         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1782         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1783
1784         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1785                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1786         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1787                 nesvnic->nic.qp_id);
1788
1789         if (++cqp_head >= nesdev->cqp.sq_size)
1790                 cqp_head = 0;
1791
1792         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1793
1794         /* Destroy NIC CQ */
1795         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1796         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1797                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1798         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1799                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1800
1801         if (++cqp_head >= nesdev->cqp.sq_size)
1802                 cqp_head = 0;
1803
1804         nesdev->cqp.sq_head = cqp_head;
1805         barrier();
1806
1807         /* Ring doorbell (2 WQEs) */
1808         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1809
1810         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1811         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1812                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1813                         cqp_head, nesdev->cqp.sq_head,
1814                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1815
1816         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1817                         NES_EVENT_TIMEOUT);
1818
1819         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1820                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1821                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1822         if (!ret) {
1823                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1824                                 nesvnic->nic.qp_id);
1825         }
1826
1827         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1828                         nesvnic->nic_pbase);
1829 }
1830
1831 /**
1832  * nes_napi_isr
1833  */
1834 int nes_napi_isr(struct nes_device *nesdev)
1835 {
1836         struct nes_adapter *nesadapter = nesdev->nesadapter;
1837         u32 int_stat;
1838
1839         if (nesdev->napi_isr_ran) {
1840                 /* interrupt status has already been read in ISR */
1841                 int_stat = nesdev->int_stat;
1842         } else {
1843                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1844                 nesdev->int_stat = int_stat;
1845                 nesdev->napi_isr_ran = 1;
1846         }
1847
1848         int_stat &= nesdev->int_req;
1849         /* iff NIC, process here, else wait for DPC */
1850         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1851                 nesdev->napi_isr_ran = 0;
1852                 nes_write32(nesdev->regs+NES_INT_STAT,
1853                                 (int_stat &
1854                                 ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1855
1856                 /* Process the CEQs */
1857                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1858
1859                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1860                                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1861                                           ((nesadapter->et_use_adaptive_rx_coalesce) &&
1862                                            (nesdev->deepcq_count > nesadapter->et_pkt_rate_low)))) ) {
1863                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1864                                 /* Enable Periodic timer interrupts */
1865                                 nesdev->int_req |= NES_INT_TIMER;
1866                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1867                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1868                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1869                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1870                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1871                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1872                         }
1873
1874                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1875                         {
1876                                 nes_nic_init_timer(nesdev);
1877                         }
1878                         /* Enable interrupts, except CEQs */
1879                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1880                 } else {
1881                         /* Enable interrupts, make sure timer is off */
1882                         nesdev->int_req &= ~NES_INT_TIMER;
1883                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1884                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1885                 }
1886                 nesdev->deepcq_count = 0;
1887                 return 1;
1888         } else {
1889                 return 0;
1890         }
1891 }
1892
1893
1894 /**
1895  * nes_dpc
1896  */
1897 void nes_dpc(unsigned long param)
1898 {
1899         struct nes_device *nesdev = (struct nes_device *)param;
1900         struct nes_adapter *nesadapter = nesdev->nesadapter;
1901         u32 counter;
1902         u32 loop_counter = 0;
1903         u32 int_status_bit;
1904         u32 int_stat;
1905         u32 timer_stat;
1906         u32 temp_int_stat;
1907         u32 intf_int_stat;
1908         u32 debug_error;
1909         u32 processed_intf_int = 0;
1910         u16 processed_timer_int = 0;
1911         u16 completion_ints = 0;
1912         u16 timer_ints = 0;
1913
1914         /* nes_debug(NES_DBG_ISR, "\n"); */
1915
1916         do {
1917                 timer_stat = 0;
1918                 if (nesdev->napi_isr_ran) {
1919                         nesdev->napi_isr_ran = 0;
1920                         int_stat = nesdev->int_stat;
1921                 } else
1922                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1923                 if (processed_intf_int != 0)
1924                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
1925                 else
1926                         int_stat &= nesdev->int_req;
1927                 if (processed_timer_int == 0) {
1928                         processed_timer_int = 1;
1929                         if (int_stat & NES_INT_TIMER) {
1930                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1931                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
1932                                         int_stat &= ~NES_INT_TIMER;
1933                                 }
1934                         }
1935                 } else {
1936                         int_stat &= ~NES_INT_TIMER;
1937                 }
1938
1939                 if (int_stat) {
1940                         if (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1941                                         NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)) {
1942                                 /* Ack the interrupts */
1943                                 nes_write32(nesdev->regs+NES_INT_STAT,
1944                                                 (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1945                                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1946                         }
1947
1948                         temp_int_stat = int_stat;
1949                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1950                                 if (int_stat & int_status_bit) {
1951                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1952                                         temp_int_stat &= ~int_status_bit;
1953                                         completion_ints = 1;
1954                                 }
1955                                 if (!(temp_int_stat & 0x0000ffff))
1956                                         break;
1957                                 int_status_bit <<= 1;
1958                         }
1959
1960                         /* Process the AEQ for this pci function */
1961                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1962                         if (int_stat & int_status_bit) {
1963                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1964                         }
1965
1966                         /* Process the MAC interrupt for this pci function */
1967                         int_status_bit = 1 << (24 + nesdev->mac_index);
1968                         if (int_stat & int_status_bit) {
1969                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
1970                         }
1971
1972                         if (int_stat & NES_INT_TIMER) {
1973                                 if (timer_stat & nesdev->timer_int_req) {
1974                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
1975                                                         (timer_stat & nesdev->timer_int_req) |
1976                                                         ~(nesdev->nesadapter->timer_int_req));
1977                                         timer_ints = 1;
1978                                 }
1979                         }
1980
1981                         if (int_stat & NES_INT_INTF) {
1982                                 processed_intf_int = 1;
1983                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1984                                 intf_int_stat &= nesdev->intf_int_req;
1985                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1986                                         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1987                                         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1988                                                         (u16)debug_error);
1989                                         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1990                                                         0x01010000 | (debug_error & 0x0000ffff));
1991                                         /* BUG(); */
1992                                         if (crit_err_count++ > 10)
1993                                                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1994                                 }
1995                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
1996                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
1997                                         BUG();
1998                                 }
1999                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2000                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2001                                         BUG();
2002                                 }
2003                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2004                         }
2005
2006                         if (int_stat & NES_INT_TSW) {
2007                         }
2008                 }
2009                 /* Don't use the interface interrupt bit stay in loop */
2010                 int_stat &= ~NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
2011                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3;
2012         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2013
2014         if (timer_ints == 1) {
2015                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2016                         if (completion_ints == 0) {
2017                                 nesdev->timer_only_int_count++;
2018                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2019                                         nesdev->timer_only_int_count = 0;
2020                                         nesdev->int_req &= ~NES_INT_TIMER;
2021                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2022                                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2023                                 } else {
2024                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
2025                                 }
2026                         } else {
2027                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2028                                 {
2029                                         nes_nic_init_timer(nesdev);
2030                                 }
2031                                 nesdev->timer_only_int_count = 0;
2032                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
2033                         }
2034                 } else {
2035                         nesdev->timer_only_int_count = 0;
2036                         nesdev->int_req &= ~NES_INT_TIMER;
2037                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2038                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2039                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2040                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2041                 }
2042         } else {
2043                 if ( (completion_ints == 1) &&
2044                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2045                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2046                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2047                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2048                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2049                         nesdev->timer_only_int_count = 0;
2050                         nesdev->int_req |= NES_INT_TIMER;
2051                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2052                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2053                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2054                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2055                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2056                 } else {
2057                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2058                 }
2059         }
2060         nesdev->deepcq_count = 0;
2061 }
2062
2063
2064 /**
2065  * nes_process_ceq
2066  */
2067 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2068 {
2069         u64 u64temp;
2070         struct nes_hw_cq *cq;
2071         u32 head;
2072         u32 ceq_size;
2073
2074         /* nes_debug(NES_DBG_CQ, "\n"); */
2075         head = ceq->ceq_head;
2076         ceq_size = ceq->ceq_size;
2077
2078         do {
2079                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2080                                 NES_CEQE_VALID) {
2081                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX])))<<32) |
2082                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2083                         u64temp <<= 1;
2084                         cq = *((struct nes_hw_cq **)&u64temp);
2085                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2086                         barrier();
2087                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2088
2089                         /* call the event handler */
2090                         cq->ce_handler(nesdev, cq);
2091
2092                         if (++head >= ceq_size)
2093                                 head = 0;
2094                 } else {
2095                         break;
2096                 }
2097
2098         } while (1);
2099
2100         ceq->ceq_head = head;
2101 }
2102
2103
2104 /**
2105  * nes_process_aeq
2106  */
2107 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2108 {
2109 //      u64 u64temp;
2110         u32 head;
2111         u32 aeq_size;
2112         u32 aeqe_misc;
2113         u32 aeqe_cq_id;
2114         struct nes_hw_aeqe volatile *aeqe;
2115
2116         head = aeq->aeq_head;
2117         aeq_size = aeq->aeq_size;
2118
2119         do {
2120                 aeqe = &aeq->aeq_vbase[head];
2121                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2122                         break;
2123                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2124                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2125                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2126                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2127                                 /* dealing with an accelerated QP related AE */
2128 //                              u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])))<<32) |
2129 //                                      ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2130                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2131                         } else {
2132                                 /* TODO: dealing with a CQP related AE */
2133                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2134                                                 (u16)(aeqe_misc >> 16));
2135                         }
2136                 }
2137
2138                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2139
2140                 if (++head >= aeq_size)
2141                         head = 0;
2142         }
2143         while (1);
2144         aeq->aeq_head = head;
2145 }
2146
2147 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2148 {
2149         struct nes_adapter *nesadapter = nesdev->nesadapter;
2150         u32 reset_value;
2151         u32 i=0;
2152         u32 u32temp;
2153
2154         if (nesadapter->hw_rev == NE020_REV) {
2155                 return;
2156         }
2157         mh_detected++;
2158
2159         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2160
2161         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2162                 reset_value |= 0x0000001d;
2163         else
2164                 reset_value |= 0x0000002d;
2165
2166         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2167                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2168                         nesadapter->link_interrupt_count[0] = 0;
2169                         nesadapter->link_interrupt_count[1] = 0;
2170                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2171                         if (0x00000040 & u32temp)
2172                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2173                         else
2174                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2175
2176                         reset_value |= 0x0000003d;
2177                 }
2178                 nesadapter->link_interrupt_count[mac_index] = 0;
2179         }
2180
2181         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2182
2183         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2184                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2185
2186         if (0x0000003d == (reset_value & 0x0000003d)) {
2187                 u32 pcs_control_status0, pcs_control_status1;
2188
2189                 for (i = 0; i < 10; i++) {
2190                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2191                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2192                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2193                              && (pcs_control_status0 & 0x00100000))
2194                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2195                                 && (pcs_control_status1 & 0x00100000)))
2196                                 continue;
2197                         else
2198                                 break;
2199                 }
2200                 if (10 == i) {
2201                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2202                         if (0x00000040 & u32temp)
2203                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2204                         else
2205                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2206
2207                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2208
2209                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2210                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2211                 }
2212         }
2213 }
2214
2215 /**
2216  * nes_process_mac_intr
2217  */
2218 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2219 {
2220         unsigned long flags;
2221         u32 pcs_control_status;
2222         struct nes_adapter *nesadapter = nesdev->nesadapter;
2223         struct nes_vnic *nesvnic;
2224         u32 mac_status;
2225         u32 mac_index = nesdev->mac_index;
2226         u32 u32temp;
2227         u16 phy_data;
2228         u16 temp_phy_data;
2229         u32 pcs_val  = 0x0f0f0000;
2230         u32 pcs_mask = 0x0f1f0000;
2231
2232         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2233         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2234                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2235                 return;
2236         }
2237         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2238         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2239
2240         /* ack the MAC interrupt */
2241         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2242         /* Clear the interrupt */
2243         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2244
2245         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2246
2247         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2248                 nesdev->link_status_interrupts++;
2249                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2250                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2251                         nes_reset_link(nesdev, mac_index);
2252                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2253                 }
2254                 /* read the PHY interrupt status register */
2255                 if (nesadapter->OneG_Mode) {
2256                         do {
2257                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2258                                                 nesadapter->phy_index[mac_index], &phy_data);
2259                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2260                                                 nesadapter->phy_index[mac_index], phy_data);
2261                         } while (phy_data&0x8000);
2262
2263                         temp_phy_data = 0;
2264                         do {
2265                                 nes_read_1G_phy_reg(nesdev, 0x11,
2266                                                 nesadapter->phy_index[mac_index], &phy_data);
2267                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2268                                                 nesadapter->phy_index[mac_index], phy_data);
2269                                 if (temp_phy_data == phy_data)
2270                                         break;
2271                                 temp_phy_data = phy_data;
2272                         } while (1);
2273
2274                         nes_read_1G_phy_reg(nesdev, 0x1e,
2275                                         nesadapter->phy_index[mac_index], &phy_data);
2276                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2277                                         nesadapter->phy_index[mac_index], phy_data);
2278
2279                         nes_read_1G_phy_reg(nesdev, 1,
2280                                         nesadapter->phy_index[mac_index], &phy_data);
2281                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2282                                         nesadapter->phy_index[mac_index], phy_data);
2283
2284                         if (temp_phy_data & 0x1000) {
2285                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2286                                 phy_data = 4;
2287                         } else {
2288                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2289                         }
2290                 }
2291                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2292                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2293                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2294
2295                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2296                         switch (mac_index) {
2297                         case 1:
2298                         case 3:
2299                                 pcs_control_status = nes_read_indexed(nesdev,
2300                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2301                                 break;
2302                         default:
2303                                 pcs_control_status = nes_read_indexed(nesdev,
2304                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2305                                 break;
2306                         }
2307                 } else {
2308                         pcs_control_status = nes_read_indexed(nesdev,
2309                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2310                         pcs_control_status = nes_read_indexed(nesdev,
2311                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2312                 }
2313
2314                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2315                                 mac_index, pcs_control_status);
2316                 if ((nesadapter->OneG_Mode) &&
2317                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2318                         u32temp = 0x01010000;
2319                         if (nesadapter->port_count > 2) {
2320                                 u32temp |= 0x02020000;
2321                         }
2322                         if ((pcs_control_status & u32temp)!= u32temp) {
2323                                 phy_data = 0;
2324                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2325                         }
2326                 } else {
2327                         switch (nesadapter->phy_type[mac_index]) {
2328                         case NES_PHY_TYPE_IRIS:
2329                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2330                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2331                                 u32temp = 20;
2332                                 do {
2333                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2334                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2335                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2336                                                 break;
2337                                         temp_phy_data = phy_data;
2338                                 } while (1);
2339                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2340                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2341                                 break;
2342
2343                         case NES_PHY_TYPE_ARGUS:
2344                                 /* clear the alarms */
2345                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2346                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2347                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2348                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2349                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2350                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2351                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2352                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2353                                 /* check link status */
2354                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2355                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2356                                 u32temp = 100;
2357                                 do {
2358                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2359
2360                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2361                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2362                                                 break;
2363                                         temp_phy_data = phy_data;
2364                                 } while (1);
2365                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2366                                         __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2367                                 break;
2368
2369                         case NES_PHY_TYPE_PUMA_1G:
2370                                 if (mac_index < 2)
2371                                         pcs_val = pcs_mask = 0x01010000;
2372                                 else
2373                                         pcs_val = pcs_mask = 0x02020000;
2374                                 /* fall through */
2375                         default:
2376                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2377                                 break;
2378                         }
2379                 }
2380
2381                 if (phy_data & 0x0004) {
2382                         nesadapter->mac_link_down[mac_index] = 0;
2383                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2384                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2385                                                 nesvnic->linkup);
2386                                 if (nesvnic->linkup == 0) {
2387                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2388                                                         nesvnic->netdev->name, nesvnic->netdev);
2389                                         if (netif_queue_stopped(nesvnic->netdev))
2390                                                 netif_start_queue(nesvnic->netdev);
2391                                         nesvnic->linkup = 1;
2392                                         netif_carrier_on(nesvnic->netdev);
2393                                 }
2394                         }
2395                 } else {
2396                         nesadapter->mac_link_down[mac_index] = 1;
2397                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2398                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2399                                                 nesvnic->linkup);
2400                                 if (nesvnic->linkup == 1) {
2401                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2402                                                         nesvnic->netdev->name, nesvnic->netdev);
2403                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2404                                                 netif_stop_queue(nesvnic->netdev);
2405                                         nesvnic->linkup = 0;
2406                                         netif_carrier_off(nesvnic->netdev);
2407                                 }
2408                         }
2409                 }
2410         }
2411
2412         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2413 }
2414
2415
2416
2417 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2418 {
2419         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2420
2421         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2422 }
2423
2424
2425 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2426 * getting out of nic_ce_handler
2427 */
2428 #define MAX_RQES_TO_PROCESS     384
2429
2430 /**
2431  * nes_nic_ce_handler
2432  */
2433 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2434 {
2435         u64 u64temp;
2436         dma_addr_t bus_address;
2437         struct nes_hw_nic *nesnic;
2438         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2439         struct nes_adapter *nesadapter = nesdev->nesadapter;
2440         struct nes_hw_nic_rq_wqe *nic_rqe;
2441         struct nes_hw_nic_sq_wqe *nic_sqe;
2442         struct sk_buff *skb;
2443         struct sk_buff *rx_skb;
2444         __le16 *wqe_fragment_length;
2445         u32 head;
2446         u32 cq_size;
2447         u32 rx_pkt_size;
2448         u32 cqe_count=0;
2449         u32 cqe_errv;
2450         u32 cqe_misc;
2451         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2452         u16 vlan_tag;
2453         u16 pkt_type;
2454         u16 rqes_processed = 0;
2455         u8 sq_cqes = 0;
2456         u8 nes_use_lro = 0;
2457
2458         head = cq->cq_head;
2459         cq_size = cq->cq_size;
2460         cq->cqes_pending = 1;
2461         if (nesvnic->netdev->features & NETIF_F_LRO)
2462                 nes_use_lro = 1;
2463         do {
2464                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2465                                 NES_NIC_CQE_VALID) {
2466                         nesnic = &nesvnic->nic;
2467                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2468                         if (cqe_misc & NES_NIC_CQE_SQ) {
2469                                 sq_cqes++;
2470                                 wqe_fragment_index = 1;
2471                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2472                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2473                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2474                                 /* bump past the vlan tag */
2475                                 wqe_fragment_length++;
2476                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2477                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2478                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2479                                         bus_address = (dma_addr_t)u64temp;
2480                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2481                                                 pci_unmap_single(nesdev->pcidev,
2482                                                                 bus_address,
2483                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2484                                                                 PCI_DMA_TODEVICE);
2485                                         }
2486                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2487                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2488                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2489                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2490                                                         bus_address = (dma_addr_t)u64temp;
2491                                                         pci_unmap_page(nesdev->pcidev,
2492                                                                         bus_address,
2493                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2494                                                                         PCI_DMA_TODEVICE);
2495                                                 } else
2496                                                         break;
2497                                         }
2498                                         if (skb)
2499                                                 dev_kfree_skb_any(skb);
2500                                 }
2501                                 nesnic->sq_tail++;
2502                                 nesnic->sq_tail &= nesnic->sq_size-1;
2503                                 if (sq_cqes > 128) {
2504                                         barrier();
2505                                 /* restart the queue if it had been stopped */
2506                                 if (netif_queue_stopped(nesvnic->netdev))
2507                                         netif_wake_queue(nesvnic->netdev);
2508                                         sq_cqes = 0;
2509                                 }
2510                         } else {
2511                                 rqes_processed ++;
2512
2513                                 cq->rx_cqes_completed++;
2514                                 cq->rx_pkts_indicated++;
2515                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2516                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2517                                 /* Get the skb */
2518                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2519                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2520                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2521                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2522                                 pci_unmap_single(nesdev->pcidev, bus_address,
2523                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2524                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2525                                 /* rx_skb->len = rx_pkt_size; */
2526                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2527                                 skb_put(rx_skb, rx_pkt_size);
2528                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2529                                 nesnic->rq_tail++;
2530                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2531
2532                                 atomic_inc(&nesvnic->rx_skbs_needed);
2533                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2534                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2535                                                         cq->cq_number | (cqe_count << 16));
2536 //                                      nesadapter->tune_timer.cq_count += cqe_count;
2537                                         nesdev->currcq_count += cqe_count;
2538                                         cqe_count = 0;
2539                                         nes_replenish_nic_rq(nesvnic);
2540                                 }
2541                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2542                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2543                                 rx_skb->ip_summed = CHECKSUM_NONE;
2544
2545                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2546                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2547                                         if ((cqe_errv &
2548                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2549                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2550                                                 if (nesvnic->rx_checksum_disabled == 0) {
2551                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2552                                                 }
2553                                         } else
2554                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2555                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2556                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2557
2558                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2559                                         if ((cqe_errv &
2560                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2561                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2562                                                 if (nesvnic->rx_checksum_disabled == 0) {
2563                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2564                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2565                                                                   nesvnic->netdev->name); */
2566                                                 }
2567                                         } else
2568                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2569                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2570                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2571                                         }
2572                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2573                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2574
2575                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2576                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2577                                 } else {
2578                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2579                                                 vlan_tag = (u16)(le32_to_cpu(
2580                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2581                                                                 >> 16);
2582                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2583                                                                 nesvnic->netdev->name, vlan_tag);
2584                                                 if (nes_use_lro)
2585                                                         lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2586                                                                         nesvnic->vlan_grp, vlan_tag, NULL);
2587                                                 else
2588                                                         nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2589                                         } else {
2590                                                 if (nes_use_lro)
2591                                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2592                                                 else
2593                                                         nes_netif_rx(rx_skb);
2594                                         }
2595                                 }
2596
2597                                 nesvnic->netdev->last_rx = jiffies;
2598                                 /* nesvnic->netstats.rx_packets++; */
2599                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2600                         }
2601
2602                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2603                         /* Accounting... */
2604                         cqe_count++;
2605                         if (++head >= cq_size)
2606                                 head = 0;
2607                         if (cqe_count == 255) {
2608                                 /* Replenish Nic CQ */
2609                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2610                                                 cq->cq_number | (cqe_count << 16));
2611 //                              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2612                                 nesdev->currcq_count += cqe_count;
2613                                 cqe_count = 0;
2614                         }
2615
2616                         if (cq->rx_cqes_completed >= nesvnic->budget)
2617                                 break;
2618                 } else {
2619                         cq->cqes_pending = 0;
2620                         break;
2621                 }
2622
2623         } while (1);
2624
2625         if (nes_use_lro)
2626                 lro_flush_all(&nesvnic->lro_mgr);
2627         if (sq_cqes) {
2628                 barrier();
2629                 /* restart the queue if it had been stopped */
2630                 if (netif_queue_stopped(nesvnic->netdev))
2631                         netif_wake_queue(nesvnic->netdev);
2632         }
2633         cq->cq_head = head;
2634         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2635                         cq->cq_number, cqe_count, cq->cq_head); */
2636         cq->cqe_allocs_pending = cqe_count;
2637         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2638         {
2639 //              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2640                 nesdev->currcq_count += cqe_count;
2641                 nes_nic_tune_timer(nesdev);
2642         }
2643         if (atomic_read(&nesvnic->rx_skbs_needed))
2644                 nes_replenish_nic_rq(nesvnic);
2645 }
2646
2647
2648 /**
2649  * nes_cqp_ce_handler
2650  */
2651 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2652 {
2653         u64 u64temp;
2654         unsigned long flags;
2655         struct nes_hw_cqp *cqp = NULL;
2656         struct nes_cqp_request *cqp_request;
2657         struct nes_hw_cqp_wqe *cqp_wqe;
2658         u32 head;
2659         u32 cq_size;
2660         u32 cqe_count=0;
2661         u32 error_code;
2662         /* u32 counter; */
2663
2664         head = cq->cq_head;
2665         cq_size = cq->cq_size;
2666
2667         do {
2668                 /* process the CQE */
2669                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2670                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2671
2672                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2673                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2674                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
2675                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2676                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2677                         cqp = *((struct nes_hw_cqp **)&u64temp);
2678
2679                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2680                         if (error_code) {
2681                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2682                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2683                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2684                                                 (u16)(error_code >> 16),
2685                                                 (u16)error_code);
2686                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2687                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2688                         }
2689
2690                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2691                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX])))<<32) |
2692                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2693                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2694                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2695                         if (cqp_request) {
2696                                 if (cqp_request->waiting) {
2697                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2698                                         cqp_request->major_code = (u16)(error_code >> 16);
2699                                         cqp_request->minor_code = (u16)error_code;
2700                                         barrier();
2701                                         cqp_request->request_done = 1;
2702                                         wake_up(&cqp_request->waitq);
2703                                         if (atomic_dec_and_test(&cqp_request->refcount)) {
2704                                                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2705                                                                 cqp_request,
2706                                                                 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2707                                                 if (cqp_request->dynamic) {
2708                                                         kfree(cqp_request);
2709                                                 } else {
2710                                                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2711                                                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2712                                                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2713                                                 }
2714                                         }
2715                                 } else if (cqp_request->callback) {
2716                                         /* Envoke the callback routine */
2717                                         cqp_request->cqp_callback(nesdev, cqp_request);
2718                                         if (cqp_request->dynamic) {
2719                                                 kfree(cqp_request);
2720                                         } else {
2721                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2722                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2723                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2724                                         }
2725                                 } else {
2726                                         nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2727                                                         cqp_request,
2728                                                         le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2729                                         if (cqp_request->dynamic) {
2730                                                 kfree(cqp_request);
2731                                         } else {
2732                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2733                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2734                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2735                                         }
2736                                 }
2737                         } else {
2738                                 wake_up(&nesdev->cqp.waitq);
2739                         }
2740
2741                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2742                         nes_write32(nesdev->regs+NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2743                         if (++cqp->sq_tail >= cqp->sq_size)
2744                                 cqp->sq_tail = 0;
2745
2746                         /* Accounting... */
2747                         cqe_count++;
2748                         if (++head >= cq_size)
2749                                 head = 0;
2750                 } else {
2751                         break;
2752                 }
2753         } while (1);
2754         cq->cq_head = head;
2755
2756         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2757         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2758                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2759                         (nesdev->cqp.sq_size - 1)) != 1)) {
2760                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2761                                 struct nes_cqp_request, list);
2762                 list_del_init(&cqp_request->list);
2763                 head = nesdev->cqp.sq_head++;
2764                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2765                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2766                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2767                 barrier();
2768                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2769                         cpu_to_le32((u32)((unsigned long)cqp_request));
2770                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2771                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2772                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2773                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2774                 /* Ring doorbell (1 WQEs) */
2775                 barrier();
2776                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2777         }
2778         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2779
2780         /* Arm the CCQ */
2781         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2782                         cq->cq_number);
2783         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2784 }
2785
2786
2787 /**
2788  * nes_process_iwarp_aeqe
2789  */
2790 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2791                                    struct nes_hw_aeqe *aeqe)
2792 {
2793         u64 context;
2794         u64 aeqe_context = 0;
2795         unsigned long flags;
2796         struct nes_qp *nesqp;
2797         int resource_allocated;
2798         /* struct iw_cm_id *cm_id; */
2799         struct nes_adapter *nesadapter = nesdev->nesadapter;
2800         struct ib_event ibevent;
2801         /* struct iw_cm_event cm_event; */
2802         u32 aeq_info;
2803         u32 next_iwarp_state = 0;
2804         u16 async_event_id;
2805         u8 tcp_state;
2806         u8 iwarp_state;
2807
2808         nes_debug(NES_DBG_AEQ, "\n");
2809         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2810         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2811                 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2812                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2813         } else {
2814                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2815                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2816                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2817                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2818                 BUG_ON(!context);
2819         }
2820
2821         async_event_id = (u16)aeq_info;
2822         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2823         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2824         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2825                         " Tcp state = %s, iWARP state = %s\n",
2826                         async_event_id,
2827                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2828                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2829
2830
2831         switch (async_event_id) {
2832                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2833                         nesqp = *((struct nes_qp **)&context);
2834                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2835                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2836                                 nes_add_ref(&nesqp->ibqp);
2837                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2838                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2839                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2840                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2841                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2842                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2843                                                 async_event_id, nesqp->last_aeq, tcp_state);
2844                         }
2845                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2846                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2847                                 /* FIN Received but tcp state or IB state moved on,
2848                                                 should expect a close complete */
2849                                 return;
2850                         }
2851                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2852                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2853                 case NES_AEQE_AEID_TERMINATE_SENT:
2854                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2855                 case NES_AEQE_AEID_RESET_SENT:
2856                         nesqp = *((struct nes_qp **)&context);
2857                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2858                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2859                         }
2860                         nes_add_ref(&nesqp->ibqp);
2861                         spin_lock_irqsave(&nesqp->lock, flags);
2862                         nesqp->hw_iwarp_state = iwarp_state;
2863                         nesqp->hw_tcp_state = tcp_state;
2864                         nesqp->last_aeq = async_event_id;
2865
2866                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2867                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2868                                 nesqp->hte_added = 0;
2869                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2870                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2871                                                 nesqp->hwqp.qp_id);
2872                                 nes_hw_modify_qp(nesdev, nesqp,
2873                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2874                                 spin_lock_irqsave(&nesqp->lock, flags);
2875                         }
2876
2877                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2878                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2879                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2880                                 switch (nesqp->hw_iwarp_state) {
2881                                         case NES_AEQE_IWARP_STATE_RTS:
2882                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2883                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2884                                                 break;
2885                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2886                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2887                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2888                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2889                                                         next_iwarp_state |= 0x02000000;
2890                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2891                                                 }
2892                                                 break;
2893                                         default:
2894                                                 next_iwarp_state = 0;
2895                                 }
2896                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2897                                 if (next_iwarp_state) {
2898                                         nes_add_ref(&nesqp->ibqp);
2899                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2900                                                         " also added another reference\n",
2901                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2902                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2903                                 }
2904                                 nes_cm_disconn(nesqp);
2905                         } else {
2906                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2907                                         /* FIN Received but ib state not RTS,
2908                                                         close complete will be on its way */
2909                                         spin_unlock_irqrestore(&nesqp->lock, flags);
2910                                         nes_rem_ref(&nesqp->ibqp);
2911                                         return;
2912                                 }
2913                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2914                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2915                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2916                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2917                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2918                                                         " also added another reference\n",
2919                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2920                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2921                                 }
2922                                 nes_cm_disconn(nesqp);
2923                         }
2924                         break;
2925                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2926                         nesqp = *((struct nes_qp **)&context);
2927                         spin_lock_irqsave(&nesqp->lock, flags);
2928                         nesqp->hw_iwarp_state = iwarp_state;
2929                         nesqp->hw_tcp_state = tcp_state;
2930                         nesqp->last_aeq = async_event_id;
2931                         spin_unlock_irqrestore(&nesqp->lock, flags);
2932                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2933                                         " event on QP%u \n  Q2 Data:\n",
2934                                         nesqp->hwqp.qp_id);
2935                         if (nesqp->ibqp.event_handler) {
2936                                 ibevent.device = nesqp->ibqp.device;
2937                                 ibevent.element.qp = &nesqp->ibqp;
2938                                 ibevent.event = IB_EVENT_QP_FATAL;
2939                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2940                         }
2941                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2942                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
2943                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2944                                 nes_add_ref(&nesqp->ibqp);
2945                                 nes_cm_disconn(nesqp);
2946                         } else {
2947                                 nesqp->in_disconnect = 0;
2948                                 wake_up(&nesqp->kick_waitq);
2949                         }
2950                         break;
2951                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2952                         nesqp = *((struct nes_qp **)&context);
2953                         nes_add_ref(&nesqp->ibqp);
2954                         spin_lock_irqsave(&nesqp->lock, flags);
2955                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2956                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2957                         nesqp->last_aeq = async_event_id;
2958                         if (nesqp->cm_id) {
2959                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2960                                                 " event on QP%u, remote IP = 0x%08X \n",
2961                                                 nesqp->hwqp.qp_id,
2962                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2963                         } else {
2964                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2965                                                 " event on QP%u \n",
2966                                                 nesqp->hwqp.qp_id);
2967                         }
2968                         spin_unlock_irqrestore(&nesqp->lock, flags);
2969                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2970                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2971                         if (nesqp->ibqp.event_handler) {
2972                                 ibevent.device = nesqp->ibqp.device;
2973                                 ibevent.element.qp = &nesqp->ibqp;
2974                                 ibevent.event = IB_EVENT_QP_FATAL;
2975                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2976                         }
2977                         break;
2978                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2979                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2980                                 nesqp = nesadapter->qp_table[le32_to_cpu(
2981                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2982                         } else {
2983                                 /* TODO: get the actual WQE and mask off wqe index */
2984                                 context &= ~((u64)511);
2985                                 nesqp = *((struct nes_qp **)&context);
2986                         }
2987                         spin_lock_irqsave(&nesqp->lock, flags);
2988                         nesqp->hw_iwarp_state = iwarp_state;
2989                         nesqp->hw_tcp_state = tcp_state;
2990                         nesqp->last_aeq = async_event_id;
2991                         spin_unlock_irqrestore(&nesqp->lock, flags);
2992                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
2993                                         nesqp->hwqp.qp_id);
2994                         if (nesqp->ibqp.event_handler) {
2995                                 ibevent.device = nesqp->ibqp.device;
2996                                 ibevent.element.qp = &nesqp->ibqp;
2997                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2998                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2999                         }
3000                         break;
3001                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3002                         nesqp = *((struct nes_qp **)&context);
3003                         spin_lock_irqsave(&nesqp->lock, flags);
3004                         nesqp->hw_iwarp_state = iwarp_state;
3005                         nesqp->hw_tcp_state = tcp_state;
3006                         nesqp->last_aeq = async_event_id;
3007                         spin_unlock_irqrestore(&nesqp->lock, flags);
3008                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3009                                         nesqp->hwqp.qp_id);
3010                         if (nesqp->ibqp.event_handler) {
3011                                 ibevent.device = nesqp->ibqp.device;
3012                                 ibevent.element.qp = &nesqp->ibqp;
3013                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3014                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3015                         }
3016                         break;
3017                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3018                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3019                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3020                         spin_lock_irqsave(&nesqp->lock, flags);
3021                         nesqp->hw_iwarp_state = iwarp_state;
3022                         nesqp->hw_tcp_state = tcp_state;
3023                         nesqp->last_aeq = async_event_id;
3024                         spin_unlock_irqrestore(&nesqp->lock, flags);
3025                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3026                                         " nesqp = %p, AE reported %p\n",
3027                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3028                         if (nesqp->ibqp.event_handler) {
3029                                 ibevent.device = nesqp->ibqp.device;
3030                                 ibevent.element.qp = &nesqp->ibqp;
3031                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3032                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3033                         }
3034                         break;
3035                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3036                         context <<= 1;
3037                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3038                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3039                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3040                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3041                         if (resource_allocated) {
3042                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3043                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3044                         }
3045                         break;
3046                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3047                         nesqp = nesadapter->qp_table[le32_to_cpu(
3048                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3049                         spin_lock_irqsave(&nesqp->lock, flags);
3050                         nesqp->hw_iwarp_state = iwarp_state;
3051                         nesqp->hw_tcp_state = tcp_state;
3052                         nesqp->last_aeq = async_event_id;
3053                         spin_unlock_irqrestore(&nesqp->lock, flags);
3054                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3055                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3056                                         nesqp->hwqp.qp_id);
3057                         if (nesqp->ibqp.event_handler) {
3058                                 ibevent.device = nesqp->ibqp.device;
3059                                 ibevent.element.qp = &nesqp->ibqp;
3060                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3061                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3062                         }
3063                         /* tell cm to disconnect, cm will queue work to thread */
3064                         nes_add_ref(&nesqp->ibqp);
3065                         nes_cm_disconn(nesqp);
3066                         break;
3067                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3068                         nesqp = *((struct nes_qp **)&context);
3069                         spin_lock_irqsave(&nesqp->lock, flags);
3070                         nesqp->hw_iwarp_state = iwarp_state;
3071                         nesqp->hw_tcp_state = tcp_state;
3072                         nesqp->last_aeq = async_event_id;
3073                         spin_unlock_irqrestore(&nesqp->lock, flags);
3074                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3075                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3076                                         nesqp->hwqp.qp_id);
3077                         if (nesqp->ibqp.event_handler) {
3078                                 ibevent.device = nesqp->ibqp.device;
3079                                 ibevent.element.qp = &nesqp->ibqp;
3080                                 ibevent.event = IB_EVENT_QP_FATAL;
3081                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3082                         }
3083                         /* tell cm to disconnect, cm will queue work to thread */
3084                         nes_add_ref(&nesqp->ibqp);
3085                         nes_cm_disconn(nesqp);
3086                         break;
3087                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3088                         nesqp = *((struct nes_qp **)&context);
3089                         spin_lock_irqsave(&nesqp->lock, flags);
3090                         nesqp->hw_iwarp_state = iwarp_state;
3091                         nesqp->hw_tcp_state = tcp_state;
3092                         nesqp->last_aeq = async_event_id;
3093                         spin_unlock_irqrestore(&nesqp->lock, flags);
3094                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3095                                         " event on QP%u \n  Q2 Data:\n",
3096                                         nesqp->hwqp.qp_id);
3097                         if (nesqp->ibqp.event_handler) {
3098                                 ibevent.device = nesqp->ibqp.device;
3099                                 ibevent.element.qp = &nesqp->ibqp;
3100                                 ibevent.event = IB_EVENT_QP_FATAL;
3101                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3102                         }
3103                         /* tell cm to disconnect, cm will queue work to thread */
3104                         nes_add_ref(&nesqp->ibqp);
3105                         nes_cm_disconn(nesqp);
3106                         break;
3107                         /* TODO: additional AEs need to be here */
3108                 default:
3109                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3110                                         async_event_id);
3111                         break;
3112         }
3113
3114 }
3115
3116
3117 /**
3118  * nes_iwarp_ce_handler
3119  */
3120 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3121 {
3122         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3123
3124         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3125                         nescq->hw_cq.cq_number); */
3126         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3127
3128         if (nescq->ibcq.comp_handler)
3129                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3130
3131         return;
3132 }
3133
3134
3135 /**
3136  * nes_manage_apbvt()
3137  */
3138 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3139                 u32 nic_index, u32 add_port)
3140 {
3141         struct nes_device *nesdev = nesvnic->nesdev;
3142         struct nes_hw_cqp_wqe *cqp_wqe;
3143         unsigned long flags;
3144         struct nes_cqp_request *cqp_request;
3145         int ret = 0;
3146         u16 major_code;
3147
3148         /* Send manage APBVT request to CQP */
3149         cqp_request = nes_get_cqp_request(nesdev);
3150         if (cqp_request == NULL) {
3151                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3152                 return -ENOMEM;
3153         }
3154         cqp_request->waiting = 1;
3155         cqp_wqe = &cqp_request->cqp_wqe;
3156
3157         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3158                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3159                         accel_local_port, accel_local_port, nic_index);
3160
3161         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3162         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3163                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3164         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3165                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3166
3167         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3168
3169         atomic_set(&cqp_request->refcount, 2);
3170         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3171
3172         if (add_port == NES_MANAGE_APBVT_ADD)
3173                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3174                                 NES_EVENT_TIMEOUT);
3175         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3176                         ret, cqp_request->major_code, cqp_request->minor_code);
3177         major_code = cqp_request->major_code;
3178         if (atomic_dec_and_test(&cqp_request->refcount)) {
3179                 if (cqp_request->dynamic) {
3180                         kfree(cqp_request);
3181                 } else {
3182                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3183                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3184                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3185                 }
3186         }
3187         if (!ret)
3188                 return -ETIME;
3189         else if (major_code)
3190                 return -EIO;
3191         else
3192                 return 0;
3193 }
3194
3195
3196 /**
3197  * nes_manage_arp_cache
3198  */
3199 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3200                 u32 ip_addr, u32 action)
3201 {
3202         struct nes_hw_cqp_wqe *cqp_wqe;
3203         struct nes_vnic *nesvnic = netdev_priv(netdev);
3204         struct nes_device *nesdev;
3205         struct nes_cqp_request *cqp_request;
3206         int arp_index;
3207
3208         nesdev = nesvnic->nesdev;
3209         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3210         if (arp_index == -1) {
3211                 return;
3212         }
3213
3214         /* update the ARP entry */
3215         cqp_request = nes_get_cqp_request(nesdev);
3216         if (cqp_request == NULL) {
3217                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3218                 return;
3219         }
3220         cqp_request->waiting = 0;
3221         cqp_wqe = &cqp_request->cqp_wqe;
3222         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3223
3224         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3225                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3226         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3227                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3228         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3229
3230         if (action == NES_ARP_ADD) {
3231                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3232                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3233                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3234                                 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3235                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3236                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3237         } else {
3238                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3239                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3240         }
3241
3242         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3243                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3244
3245         atomic_set(&cqp_request->refcount, 1);
3246         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3247 }
3248
3249
3250 /**
3251  * flush_wqes
3252  */
3253 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3254                 u32 which_wq, u32 wait_completion)
3255 {
3256         unsigned long flags;
3257         struct nes_cqp_request *cqp_request;
3258         struct nes_hw_cqp_wqe *cqp_wqe;
3259         int ret;
3260
3261         cqp_request = nes_get_cqp_request(nesdev);
3262         if (cqp_request == NULL) {
3263                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3264                 return;
3265         }
3266         if (wait_completion) {
3267                 cqp_request->waiting = 1;
3268                 atomic_set(&cqp_request->refcount, 2);
3269         } else {
3270                 cqp_request->waiting = 0;
3271         }
3272         cqp_wqe = &cqp_request->cqp_wqe;
3273         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3274
3275         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3276                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3277         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3278
3279         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3280
3281         if (wait_completion) {
3282                 /* Wait for CQP */
3283                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3284                                 NES_EVENT_TIMEOUT);
3285                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3286                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3287                                 ret, cqp_request->major_code, cqp_request->minor_code);
3288                 if (atomic_dec_and_test(&cqp_request->refcount)) {
3289                         if (cqp_request->dynamic) {
3290                                 kfree(cqp_request);
3291                         } else {
3292                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3293                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3294                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3295                         }
3296                 }
3297         }
3298 }