SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel-NE, Inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static u32 crit_err_count;
  50 u32 int_mod_timer_init;
  51 u32 int_mod_cq_depth_256;
  52 u32 int_mod_cq_depth_128;
  53 u32 int_mod_cq_depth_32;
  54 u32 int_mod_cq_depth_24;
  55 u32 int_mod_cq_depth_16;
  56 u32 int_mod_cq_depth_4;
  57 u32 int_mod_cq_depth_1;
  58 static const u8 nes_max_critical_error_count = 100;
  59 #include "nes_cm.h"
  60
  61 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  62 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  63 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  64                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  65 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  66 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  67 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  68 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  69                                    struct nes_hw_aeqe *aeqe);
  70 static void process_critical_error(struct nes_device *nesdev);
  71 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  72 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  73
  74 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  75 static unsigned char *nes_iwarp_state_str[] = {
  76         "Non-Existant",
  77         "Idle",
  78         "RTS",
  79         "Closing",
  80         "RSVD1",
  81         "Terminate",
  82         "Error",
  83         "RSVD2",
  84 };
  85
  86 static unsigned char *nes_tcp_state_str[] = {
  87         "Non-Existant",
  88         "Closed",
  89         "Listen",
  90         "SYN Sent",
  91         "SYN Rcvd",
  92         "Established",
  93         "Close Wait",
  94         "FIN Wait 1",
  95         "Closing",
  96         "Last Ack",
  97         "FIN Wait 2",
  98         "Time Wait",
  99         "RSVD1",
 100         "RSVD2",
 101         "RSVD3",
 102         "RSVD4",
 103 };
 104 #endif
 105
 106
 107 /**
 108  * nes_nic_init_timer_defaults
 109  */
 110 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 111 {
 112         unsigned long flags;
 113         struct nes_adapter *nesadapter = nesdev->nesadapter;
 114         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 115
 116         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 117
 118         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 119         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 120         if (jumbomode) {
 121                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 122                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 123                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 124         } else {
 125                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 126                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 127                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 128         }
 129
 130         /* todo use netdev->mtu to set thresholds */
 131         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 132 }
 133
 134
 135 /**
 136  * nes_nic_init_timer
 137  */
 138 static void  nes_nic_init_timer(struct nes_device *nesdev)
 139 {
 140         unsigned long flags;
 141         struct nes_adapter *nesadapter = nesdev->nesadapter;
 142         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 143
 144         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 145
 146         if (shared_timer->timer_in_use_old == 0) {
 147                 nesdev->deepcq_count = 0;
 148                 shared_timer->timer_direction_upward = 0;
 149                 shared_timer->timer_direction_downward = 0;
 150                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 151                 shared_timer->timer_in_use_old = 0;
 152
 153         }
 154         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 155                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 156                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 157                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 158         }
 159         /* todo use netdev->mtu to set thresholds */
 160         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 161 }
 162
 163
 164 /**
 165  * nes_nic_tune_timer
 166  */
 167 static void nes_nic_tune_timer(struct nes_device *nesdev)
 168 {
 169         unsigned long flags;
 170         struct nes_adapter *nesadapter = nesdev->nesadapter;
 171         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 172         u16 cq_count = nesdev->currcq_count;
 173
 174         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 175
 176         if (shared_timer->cq_count_old <= cq_count)
 177                 shared_timer->cq_direction_downward = 0;
 178         else
 179                 shared_timer->cq_direction_downward++;
 180         shared_timer->cq_count_old = cq_count;
 181         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 182                 if (cq_count <= shared_timer->threshold_low &&
 183                     shared_timer->threshold_low > 4) {
 184                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 185                         shared_timer->cq_direction_downward=0;
 186                         nesdev->currcq_count = 0;
 187                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 188                         return;
 189                 }
 190         }
 191
 192         if (cq_count > 1) {
 193                 nesdev->deepcq_count += cq_count;
 194                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 195                         shared_timer->timer_direction_upward++;
 196                         shared_timer->timer_direction_downward = 0;
 197                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 198                         shared_timer->timer_direction_upward = 0;
 199                         shared_timer->timer_direction_downward = 0;
 200                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 201                         shared_timer->timer_direction_downward++;
 202                         shared_timer->timer_direction_upward = 0;
 203                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 204                         shared_timer->timer_in_use -= 2;
 205                         shared_timer->timer_direction_upward = 0;
 206                         shared_timer->timer_direction_downward++;
 207                 } else {
 208                         shared_timer->timer_in_use -= 4;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward++;
 211                 }
 212
 213                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 214                         shared_timer->timer_in_use += 3;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward = 0;
 217                 }
 218                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 219                         shared_timer->timer_in_use -= 4 ;
 220                         shared_timer->timer_direction_downward = 0;
 221                         shared_timer->timer_direction_upward = 0;
 222                 }
 223         }
 224
 225         /* boundary checking */
 226         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 227                 shared_timer->timer_in_use = shared_timer->threshold_high;
 228         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 229                 shared_timer->timer_in_use = shared_timer->threshold_low;
 230
 231         nesdev->currcq_count = 0;
 232
 233         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 234 }
 235
 236
 237 /**
 238  * nes_init_adapter - initialize adapter
 239  */
 240 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 241         struct nes_adapter *nesadapter = NULL;
 242         unsigned long num_pds;
 243         u32 u32temp;
 244         u32 port_count;
 245         u16 max_rq_wrs;
 246         u16 max_sq_wrs;
 247         u32 max_mr;
 248         u32 max_256pbl;
 249         u32 max_4kpbl;
 250         u32 max_qp;
 251         u32 max_irrq;
 252         u32 max_cq;
 253         u32 hte_index_mask;
 254         u32 adapter_size;
 255         u32 arp_table_size;
 256         u16 vendor_id;
 257         u16 device_id;
 258         u8  OneG_Mode;
 259         u8  func_index;
 260
 261         /* search the list of existing adapters */
 262         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 263                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 264                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 265                                 nesdev->pcidev->devfn,
 266                                 PCI_SLOT(nesadapter->devfn),
 267                                 nesadapter->bus_number,
 268                                 PCI_SLOT(nesdev->pcidev->devfn),
 269                                 nesdev->pcidev->bus->number );
 270                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 271                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 272                         nesadapter->ref_count++;
 273                         return nesadapter;
 274                 }
 275         }
 276
 277         /* no adapter found */
 278         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 279         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 280                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 281                                 hw_rev);
 282                 return NULL;
 283         }
 284
 285         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 286                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 287                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 288                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 289                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 290
 291         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 292
 293
 294         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 295                 return NULL;
 296
 297         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 298         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 299
 300         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 301         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 302                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 303                                 max_qp, u32temp);
 304                 max_qp = (u32)1 << (u32temp & 0x001f);
 305         }
 306
 307         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 308         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 309                         max_qp, hte_index_mask);
 310
 311         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 312
 313         max_irrq = 1 << (u32temp & 0x001f);
 314
 315         if (max_qp > max_irrq) {
 316                 max_qp = max_irrq;
 317                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 318                                 max_qp);
 319         }
 320
 321         /* there should be no reason to allocate more pds than qps */
 322         if (num_pds > max_qp)
 323                 num_pds = max_qp;
 324
 325         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 326         max_mr = (u32)8192 << (u32temp & 0x7);
 327
 328         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 329         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 330         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 331         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 332
 333         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 334         arp_table_size = 1 << u32temp;
 335
 336         adapter_size = (sizeof(struct nes_adapter) +
 337                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 340         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 341         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 342         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 343         adapter_size += sizeof(struct nes_qp **) * max_qp;
 344
 345         /* allocate a new adapter struct */
 346         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 347         if (nesadapter == NULL) {
 348                 return NULL;
 349         }
 350
 351         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 352                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 353
 354         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 355                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 356                 kfree(nesadapter);
 357                 return NULL;
 358         }
 359
 360         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 361                                 (nesadapter->mac_addr_low >> 24);
 362
 363         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 364                                  PCI_DEVICE_ID, &device_id);
 365         nesadapter->vendor_part_id = device_id;
 366
 367         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 368                                                         OneG_Mode)) {
 369                 kfree(nesadapter);
 370                 return NULL;
 371         }
 372         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 373
 374         memset(nesadapter->pft_mcast_map, 255,
 375                sizeof nesadapter->pft_mcast_map);
 376
 377         /* populate the new nesadapter */
 378         nesadapter->devfn = nesdev->pcidev->devfn;
 379         nesadapter->bus_number = nesdev->pcidev->bus->number;
 380         nesadapter->ref_count = 1;
 381         nesadapter->timer_int_req = 0xffff0000;
 382         nesadapter->OneG_Mode = OneG_Mode;
 383         nesadapter->doorbell_start = nesdev->doorbell_region;
 384
 385         /* nesadapter->tick_delta = clk_divisor; */
 386         nesadapter->hw_rev = hw_rev;
 387         nesadapter->port_count = port_count;
 388
 389         nesadapter->max_qp = max_qp;
 390         nesadapter->hte_index_mask = hte_index_mask;
 391         nesadapter->max_irrq = max_irrq;
 392         nesadapter->max_mr = max_mr;
 393         nesadapter->max_256pbl = max_256pbl - 1;
 394         nesadapter->max_4kpbl = max_4kpbl - 1;
 395         nesadapter->max_cq = max_cq;
 396         nesadapter->free_256pbl = max_256pbl - 1;
 397         nesadapter->free_4kpbl = max_4kpbl - 1;
 398         nesadapter->max_pd = num_pds;
 399         nesadapter->arp_table_size = arp_table_size;
 400
 401         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 402         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 403                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 404                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 405                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 406         } else {
 407                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 408                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 409                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 410                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 411         }
 412         /* Setup and enable the periodic timer */
 413         if (nesadapter->et_rx_coalesce_usecs_irq)
 414                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 415                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 416         else
 417                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 418
 419         nesadapter->base_pd = 1;
 420
 421         nesadapter->device_cap_flags =
 422                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 423
 424         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 425                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 426         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 427         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 428         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 429         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 430         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 431
 432
 433         /* mark the usual suspect QPs and CQs as in use */
 434         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 435                 set_bit(u32temp, nesadapter->allocated_qps);
 436                 set_bit(u32temp, nesadapter->allocated_cqs);
 437         }
 438
 439         for (u32temp = 0; u32temp < 20; u32temp++)
 440                 set_bit(u32temp, nesadapter->allocated_pds);
 441         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 442
 443         max_rq_wrs = ((u32temp >> 8) & 3);
 444         switch (max_rq_wrs) {
 445                 case 0:
 446                         max_rq_wrs = 4;
 447                         break;
 448                 case 1:
 449                         max_rq_wrs = 16;
 450                         break;
 451                 case 2:
 452                         max_rq_wrs = 32;
 453                         break;
 454                 case 3:
 455                         max_rq_wrs = 512;
 456                         break;
 457         }
 458
 459         max_sq_wrs = (u32temp & 3);
 460         switch (max_sq_wrs) {
 461                 case 0:
 462                         max_sq_wrs = 4;
 463                         break;
 464                 case 1:
 465                         max_sq_wrs = 16;
 466                         break;
 467                 case 2:
 468                         max_sq_wrs = 32;
 469                         break;
 470                 case 3:
 471                         max_sq_wrs = 512;
 472                         break;
 473         }
 474         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 475         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 476
 477         nesadapter->max_sge = 4;
 478         nesadapter->max_cqe = 32767;
 479
 480         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 481                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 482                 kfree(nesadapter);
 483                 return NULL;
 484         }
 485
 486         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 487         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 488                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 489
 490         /* setup port configuration */
 491         if (nesadapter->port_count == 1) {
 492                 nesadapter->log_port = 0x00000000;
 493                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 494                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 495                 else
 496                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 497         } else {
 498                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 499                         nesadapter->log_port = 0x000000D8;
 500                 } else {
 501                         if (nesadapter->port_count == 2)
 502                                 nesadapter->log_port = 0x00000044;
 503                         else
 504                                 nesadapter->log_port = 0x000000e4;
 505                 }
 506                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 507         }
 508
 509         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 510                                                 nesadapter->log_port);
 511         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 512                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 513
 514         spin_lock_init(&nesadapter->resource_lock);
 515         spin_lock_init(&nesadapter->phy_lock);
 516         spin_lock_init(&nesadapter->pbl_lock);
 517         spin_lock_init(&nesadapter->periodic_timer_lock);
 518
 519         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 520         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 521         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 522         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 523
 524         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 525                 u32 pcs_control_status0, pcs_control_status1;
 526                 u32 reset_value;
 527                 u32 i = 0;
 528                 u32 int_cnt = 0;
 529                 u32 ext_cnt = 0;
 530                 unsigned long flags;
 531                 u32 j = 0;
 532
 533                 pcs_control_status0 = nes_read_indexed(nesdev,
 534                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 535                 pcs_control_status1 = nes_read_indexed(nesdev,
 536                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 537
 538                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 539                         pcs_control_status0 = nes_read_indexed(nesdev,
 540                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 541                         pcs_control_status1 = nes_read_indexed(nesdev,
 542                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 543                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 544                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 545                                 int_cnt++;
 546                         msleep(1);
 547                 }
 548                 if (int_cnt > 1) {
 549                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 550                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 551                         mh_detected++;
 552                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 553                         reset_value |= 0x0000003d;
 554                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 555
 556                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 557                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 558                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 559
 560                         pcs_control_status0 = nes_read_indexed(nesdev,
 561                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 562                         pcs_control_status1 = nes_read_indexed(nesdev,
 563                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 564
 565                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 566                                 pcs_control_status0 = nes_read_indexed(nesdev,
 567                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 568                                 pcs_control_status1 = nes_read_indexed(nesdev,
 569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 570                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 571                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 572                                         if (++ext_cnt > int_cnt) {
 573                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 574                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 575                                                                 0x0000F0C8);
 576                                                 mh_detected++;
 577                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 578                                                 reset_value |= 0x0000003d;
 579                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 580
 581                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 582                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 583                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 584                                                 break;
 585                                         }
 586                                 }
 587                                 msleep(1);
 588                         }
 589                 }
 590         }
 591
 592         if (nesadapter->hw_rev == NE020_REV) {
 593                 init_timer(&nesadapter->mh_timer);
 594                 nesadapter->mh_timer.function = nes_mh_fix;
 595                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 596                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 597                 add_timer(&nesadapter->mh_timer);
 598         } else {
 599                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 600         }
 601
 602         init_timer(&nesadapter->lc_timer);
 603         nesadapter->lc_timer.function = nes_clc;
 604         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 605         nesadapter->lc_timer.data = (unsigned long)nesdev;
 606         add_timer(&nesadapter->lc_timer);
 607
 608         list_add_tail(&nesadapter->list, &nes_adapter_list);
 609
 610         for (func_index = 0; func_index < 8; func_index++) {
 611                 pci_bus_read_config_word(nesdev->pcidev->bus,
 612                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 613                                         func_index), 0, &vendor_id);
 614                 if (vendor_id == 0xffff)
 615                         break;
 616         }
 617         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 618                 func_index, pci_name(nesdev->pcidev));
 619         nesadapter->adapter_fcn_count = func_index;
 620
 621         return nesadapter;
 622 }
 623
 624
 625 /**
 626  * nes_reset_adapter_ne020
 627  */
 628 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 629 {
 630         u32 port_count;
 631         u32 u32temp;
 632         u32 i;
 633
 634         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 635         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 636         /* TODO: assuming that both SERDES are set the same for now */
 637         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 638         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 639                         u32temp, port_count);
 640         if (*OneG_Mode)
 641                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 642         u32temp &= 0xff00ffc0;
 643         switch (port_count) {
 644                 case 1:
 645                         u32temp |= 0x00ee0000;
 646                         break;
 647                 case 2:
 648                         u32temp |= 0x00cc0000;
 649                         break;
 650                 case 4:
 651                         u32temp |= 0x00000000;
 652                         break;
 653                 default:
 654                         return 0;
 655                         break;
 656         }
 657
 658         /* check and do full reset if needed */
 659         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 660                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 661                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 662
 663                 i = 0;
 664                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 665                         mdelay(1);
 666                 if (i >= 10000) {
 667                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 668                         return 0;
 669                 }
 670
 671                 i = 0;
 672                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 673                         mdelay(1);
 674                 if (i >= 10000) {
 675                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 676                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 677                         return 0;
 678                 }
 679         }
 680
 681         /* port reset */
 682         switch (port_count) {
 683                 case 1:
 684                         u32temp |= 0x00ee0010;
 685                         break;
 686                 case 2:
 687                         u32temp |= 0x00cc0030;
 688                         break;
 689                 case 4:
 690                         u32temp |= 0x00000030;
 691                         break;
 692         }
 693
 694         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 695         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 696
 697         i = 0;
 698         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 699                 mdelay(1);
 700         if (i >= 10000) {
 701                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 702                 return 0;
 703         }
 704
 705         /* serdes 0 */
 706         i = 0;
 707         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 708                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 709                 mdelay(1);
 710         if (i >= 5000) {
 711                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 712                 return 0;
 713         }
 714
 715         /* serdes 1 */
 716         if (port_count > 1) {
 717                 i = 0;
 718                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 719                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 720                         mdelay(1);
 721                 if (i >= 5000) {
 722                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 723                         return 0;
 724                 }
 725         }
 726
 727         return port_count;
 728 }
 729
 730
 731 /**
 732  * nes_init_serdes
 733  */
 734 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 735                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 736 {
 737         int i;
 738         u32 u32temp;
 739         u32 serdes_common_control;
 740
 741         if (hw_rev != NE020_REV) {
 742                 /* init serdes 0 */
 743
 744                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 745                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 746                         serdes_common_control = nes_read_indexed(nesdev,
 747                                         NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 748                         serdes_common_control |= 0x000000100;
 749                         nes_write_indexed(nesdev,
 750                                         NES_IDX_ETH_SERDES_COMMON_CONTROL0,
 751                                         serdes_common_control);
 752                 } else if (!OneG_Mode) {
 753                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 754                 }
 755                 if (((port_count > 1) &&
 756                         (nesadapter->phy_type[0] != NES_PHY_TYPE_PUMA_1G)) ||
 757                         ((port_count > 2) &&
 758                         (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G))) {
 759                         /* init serdes 1 */
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 761                         if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 762                                 serdes_common_control = nes_read_indexed(nesdev,
 763                                         NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 764                                 serdes_common_control |= 0x000000100;
 765                                 nes_write_indexed(nesdev,
 766                                         NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 767                                         serdes_common_control);
 768                         } else if (!OneG_Mode) {
 769                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 770                         }
 771                 }
 772         } else {
 773                 /* init serdes 0 */
 774                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 775                 i = 0;
 776                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 777                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 778                         mdelay(1);
 779                 if (i >= 5000) {
 780                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 781                         return 1;
 782                 }
 783                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 784                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 785                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 786                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 787                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 788                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 789                 if (OneG_Mode)
 790                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 791                 else
 792                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 793
 794                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 795                 if (port_count > 1) {
 796                         /* init serdes 1 */
 797                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 798                         i = 0;
 799                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 800                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 801                                 mdelay(1);
 802                         if (i >= 5000) {
 803                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 804                                 /* return 1; */
 805                         }
 806                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 807                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 809                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 810                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 811                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 812                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 813                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 814                 }
 815         }
 816         return 0;
 817 }
 818
 819
 820 /**
 821  * nes_init_csr_ne020
 822  * Initialize registers for ne020 hardware
 823  */
 824 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 825 {
 826         u32 u32temp;
 827
 828         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 829
 830         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 831         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 832         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 833         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 834         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 835         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 836         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 837         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 838
 839         /* TODO: move these MAC register settings to NIC bringup */
 840         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 841         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 842         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 843         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 844         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 845         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 846         if (port_count > 1) {
 847                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 848                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 849                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 850                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 851                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 852                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 853                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 854         }
 855         if (port_count > 2) {
 856                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 857                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 858                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 859                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 860                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 861                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 862                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 863
 864                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 865                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 866                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 867                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 868                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 869                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 870                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 871         }
 872
 873         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 874         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 875         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 876                                                          0x00000001);
 877         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 878         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 879         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 880         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 881         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 882
 883         /* TODO: move this to code, get from EEPROM */
 884         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 885         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 886         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 887
 888         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 889         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 890
 891         if (hw_rev != NE020_REV) {
 892                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 893                 u32temp |= 0x80000000;
 894                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 895                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 896                 u32temp &= 0x7fffffff;
 897                 u32temp |= 0x7fff0010;
 898                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 899         }
 900 }
 901
 902
 903 /**
 904  * nes_destroy_adapter - destroy the adapter structure
 905  */
 906 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 907 {
 908         struct nes_adapter *tmp_adapter;
 909
 910         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 911                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 912                                 tmp_adapter);
 913         }
 914
 915         nesadapter->ref_count--;
 916         if (!nesadapter->ref_count) {
 917                 if (nesadapter->hw_rev == NE020_REV) {
 918                         del_timer(&nesadapter->mh_timer);
 919                 }
 920                 del_timer(&nesadapter->lc_timer);
 921
 922                 list_del(&nesadapter->list);
 923                 kfree(nesadapter);
 924         }
 925 }
 926
 927
 928 /**
 929  * nes_init_cqp
 930  */
 931 int nes_init_cqp(struct nes_device *nesdev)
 932 {
 933         struct nes_adapter *nesadapter = nesdev->nesadapter;
 934         struct nes_hw_cqp_qp_context *cqp_qp_context;
 935         struct nes_hw_cqp_wqe *cqp_wqe;
 936         struct nes_hw_ceq *ceq;
 937         struct nes_hw_ceq *nic_ceq;
 938         struct nes_hw_aeq *aeq;
 939         void *vmem;
 940         dma_addr_t pmem;
 941         u32 count=0;
 942         u32 cqp_head;
 943         u64 u64temp;
 944         u32 u32temp;
 945
 946         /* allocate CQP memory */
 947         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 948         /* SQ is 512 byte aligned, others are 256 byte aligned */
 949         nesdev->cqp_mem_size = 512 +
 950                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 951                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 952                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 953                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 954                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 955                         sizeof(struct nes_hw_cqp_qp_context);
 956
 957         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 958                         &nesdev->cqp_pbase);
 959         if (!nesdev->cqp_vbase) {
 960                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 961                 return -ENOMEM;
 962         }
 963         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 964
 965         /* Allocate a twice the number of CQP requests as the SQ size */
 966         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 967                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 968         if (nesdev->nes_cqp_requests == NULL) {
 969                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 970                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 971                                 nesdev->cqp.sq_pbase);
 972                 return -ENOMEM;
 973         }
 974
 975         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 976                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 977
 978         spin_lock_init(&nesdev->cqp.lock);
 979         init_waitqueue_head(&nesdev->cqp.waitq);
 980
 981         /* Setup Various Structures */
 982         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 983                         ~(unsigned long)(512 - 1));
 984         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 985                         ~(unsigned long long)(512 - 1));
 986
 987         nesdev->cqp.sq_vbase = vmem;
 988         nesdev->cqp.sq_pbase = pmem;
 989         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 990         nesdev->cqp.sq_head = 0;
 991         nesdev->cqp.sq_tail = 0;
 992         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 993
 994         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 995         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 996
 997         nesdev->ccq.cq_vbase = vmem;
 998         nesdev->ccq.cq_pbase = pmem;
 999         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1000         nesdev->ccq.cq_head = 0;
1001         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1002         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1003
1004         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1005         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1006
1007         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1008         ceq = &nesadapter->ceq[nesdev->ceq_index];
1009         ceq->ceq_vbase = vmem;
1010         ceq->ceq_pbase = pmem;
1011         ceq->ceq_size = NES_CCEQ_SIZE;
1012         ceq->ceq_head = 0;
1013
1014         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1015         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1016
1017         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1018         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1019         nic_ceq->ceq_vbase = vmem;
1020         nic_ceq->ceq_pbase = pmem;
1021         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1022         nic_ceq->ceq_head = 0;
1023
1024         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1025         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1026
1027         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1028         aeq->aeq_vbase = vmem;
1029         aeq->aeq_pbase = pmem;
1030         aeq->aeq_size = nesadapter->max_qp;
1031         aeq->aeq_head = 0;
1032
1033         /* Setup QP Context */
1034         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1035         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1036
1037         cqp_qp_context = vmem;
1038         cqp_qp_context->context_words[0] =
1039                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1040         cqp_qp_context->context_words[1] = 0;
1041         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1042         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1043
1044
1045         /* Write the address to Create CQP */
1046         if ((sizeof(dma_addr_t) > 4)) {
1047                 nes_write_indexed(nesdev,
1048                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1049                                 ((u64)pmem) >> 32);
1050         } else {
1051                 nes_write_indexed(nesdev,
1052                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1053         }
1054         nes_write_indexed(nesdev,
1055                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1056                         (u32)pmem);
1057
1058         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1059         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1060
1061         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1062                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1063                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1064         }
1065
1066         /* Write Create CCQ WQE */
1067         cqp_head = nesdev->cqp.sq_head++;
1068         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1069         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1070         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1071                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1072                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1073         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1074                             (nesdev->ccq.cq_number |
1075                              ((u32)nesdev->ceq_index << 16)));
1076         u64temp = (u64)nesdev->ccq.cq_pbase;
1077         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1078         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1079         u64temp = (unsigned long)&nesdev->ccq;
1080         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1081                         cpu_to_le32((u32)(u64temp >> 1));
1082         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1083                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1084         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1085
1086         /* Write Create CEQ WQE */
1087         cqp_head = nesdev->cqp.sq_head++;
1088         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1089         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1090         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1091                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1092         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1093         u64temp = (u64)ceq->ceq_pbase;
1094         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1095
1096         /* Write Create AEQ WQE */
1097         cqp_head = nesdev->cqp.sq_head++;
1098         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1099         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1100         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1101                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1102         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1103         u64temp = (u64)aeq->aeq_pbase;
1104         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1105
1106         /* Write Create NIC CEQ WQE */
1107         cqp_head = nesdev->cqp.sq_head++;
1108         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1109         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1110         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1111                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1112         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1113         u64temp = (u64)nic_ceq->ceq_pbase;
1114         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1115
1116         /* Poll until CCQP done */
1117         count = 0;
1118         do {
1119                 if (count++ > 1000) {
1120                         printk(KERN_ERR PFX "Error creating CQP\n");
1121                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1122                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1123                         return -1;
1124                 }
1125                 udelay(10);
1126         } while (!(nes_read_indexed(nesdev,
1127                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1128
1129         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1130                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1131
1132         u32temp = 0x04800000;
1133         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1134
1135         /* wait for the CCQ, CEQ, and AEQ to get created */
1136         count = 0;
1137         do {
1138                 if (count++ > 1000) {
1139                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1140                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1141                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1142                         return -1;
1143                 }
1144                 udelay(10);
1145         } while (((nes_read_indexed(nesdev,
1146                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1147
1148         /* dump the QP status value */
1149         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1150                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1151
1152         nesdev->cqp.sq_tail++;
1153
1154         return 0;
1155 }
1156
1157
1158 /**
1159  * nes_destroy_cqp
1160  */
1161 int nes_destroy_cqp(struct nes_device *nesdev)
1162 {
1163         struct nes_hw_cqp_wqe *cqp_wqe;
1164         u32 count = 0;
1165         u32 cqp_head;
1166         unsigned long flags;
1167
1168         do {
1169                 if (count++ > 1000)
1170                         break;
1171                 udelay(10);
1172         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1173
1174         /* Reset CCQ */
1175         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1176                         nesdev->ccq.cq_number);
1177
1178         /* Disable device interrupts */
1179         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1180
1181         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1182
1183         /* Destroy the AEQ */
1184         cqp_head = nesdev->cqp.sq_head++;
1185         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1186         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1187         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1188                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1189         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1190
1191         /* Destroy the NIC CEQ */
1192         cqp_head = nesdev->cqp.sq_head++;
1193         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1194         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1195         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1196                         ((u32)nesdev->nic_ceq_index << 8));
1197
1198         /* Destroy the CEQ */
1199         cqp_head = nesdev->cqp.sq_head++;
1200         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1201         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1202         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1203                         (nesdev->ceq_index << 8));
1204
1205         /* Destroy the CCQ */
1206         cqp_head = nesdev->cqp.sq_head++;
1207         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1208         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1209         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1210         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1211                         ((u32)nesdev->ceq_index << 16));
1212
1213         /* Destroy CQP */
1214         cqp_head = nesdev->cqp.sq_head++;
1215         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1216         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1217         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1218                         NES_CQP_QP_TYPE_CQP);
1219         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1220
1221         barrier();
1222         /* Ring doorbell (5 WQEs) */
1223         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1224
1225         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1226
1227         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1228         count = 0;
1229         do {
1230                 if (count++ > 1000) {
1231                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1232                                         PCI_FUNC(nesdev->pcidev->devfn));
1233                         break;
1234                 }
1235                 udelay(10);
1236         } while (((nes_read_indexed(nesdev,
1237                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1238
1239         /* dump the QP status value */
1240         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1241                         PCI_FUNC(nesdev->pcidev->devfn),
1242                         nes_read_indexed(nesdev,
1243                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1244
1245         kfree(nesdev->nes_cqp_requests);
1246
1247         /* Free the control structures */
1248         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1249                         nesdev->cqp.sq_pbase);
1250
1251         return 0;
1252 }
1253
1254
1255 /**
1256  * nes_init_phy
1257  */
1258 int nes_init_phy(struct nes_device *nesdev)
1259 {
1260         struct nes_adapter *nesadapter = nesdev->nesadapter;
1261         u32 counter = 0;
1262         u32 sds_common_control0;
1263         u32 mac_index = nesdev->mac_index;
1264         u32 tx_config = 0;
1265         u16 phy_data;
1266         u32 temp_phy_data = 0;
1267         u32 temp_phy_data2 = 0;
1268         u32 i = 0;
1269
1270         if ((nesadapter->OneG_Mode) &&
1271             (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
1272                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1273                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1274                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1275                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1276                         tx_config &= 0xFFFFFFE3;
1277                         tx_config |= 0x04;
1278                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1279                 }
1280
1281                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1282                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1283                                 nesadapter->phy_index[mac_index], phy_data);
1284                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index], 0xb000);
1285
1286                 /* Reset the PHY */
1287                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1288                 udelay(100);
1289                 counter = 0;
1290                 do {
1291                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1292                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1293                         if (counter++ > 100) break;
1294                 } while (phy_data & 0x8000);
1295
1296                 /* Setting no phy loopback */
1297                 phy_data &= 0xbfff;
1298                 phy_data |= 0x1140;
1299                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1300                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1301                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1302
1303                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1304                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1305
1306                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1307                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1308
1309                 /* Setting the interrupt mask */
1310                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1311                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1312                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1313
1314                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1315                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1316
1317                 /* turning on flow control */
1318                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1319                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1320                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1321                                 (phy_data & ~(0x03E0)) | 0xc00);
1322                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1323                                 phy_data | 0xc00); */
1324                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1325                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1326
1327                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1328                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1329                 /* Clear Half duplex */
1330                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1331                                 phy_data & ~(0x0100));
1332                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1333                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1334
1335                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1336                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1337         } else {
1338                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) ||
1339                     (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1340                         /* setup 10G MDIO operation */
1341                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1342                         tx_config &= 0xFFFFFFE3;
1343                         tx_config |= 0x15;
1344                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1345                 }
1346                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1347                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1348
1349                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1350                         mdelay(10);
1351                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1352                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1353
1354                         /*
1355                          * if firmware is already running (like from a
1356                          * driver un-load/load, don't do anything.
1357                          */
1358                         if (temp_phy_data == temp_phy_data2) {
1359                                 /* configure QT2505 AMCC PHY */
1360                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0x0000, 0x8000);
1361                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0000);
1362                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc302, 0x0044);
1363                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc318, 0x0052);
1364                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc319, 0x0008);
1365                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc31a, 0x0098);
1366                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0026, 0x0E00);
1367                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0027, 0x0001);
1368                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0028, 0xA528);
1369
1370                                 /*
1371                                  * remove micro from reset; chip boots from ROM,
1372                                  * uploads EEPROM f/w image, uC executes f/w
1373                                  */
1374                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0002);
1375
1376                                 /*
1377                                  * wait for heart beat to start to
1378                                  * know loading is done
1379                                  */
1380                                 counter = 0;
1381                                 do {
1382                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1383                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1384                                         if (counter++ > 1000) {
1385                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from heartbeat check <this is bad!!!> \n");
1386                                                 break;
1387                                         }
1388                                         mdelay(100);
1389                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1390                                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1391                                 } while ((temp_phy_data2 == temp_phy_data));
1392
1393                                 /*
1394                                  * wait for tracking to start to know
1395                                  * f/w is good to go
1396                                  */
1397                                 counter = 0;
1398                                 do {
1399                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7fd);
1400                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1401                                         if (counter++ > 1000) {
1402                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from status check <this is bad!!!> \n");
1403                                                 break;
1404                                         }
1405                                         mdelay(1000);
1406                                         /*
1407                                          * nes_debug(NES_DBG_PHY, "AMCC PHY- phy_status not ready yet = 0x%02X\n",
1408                                          *                      temp_phy_data);
1409                                          */
1410                                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1411
1412                                 /* set LOS Control invert RXLOSB_I_PADINV */
1413                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd003, 0x0000);
1414                                 /* set LOS Control to mask of RXLOSB_I */
1415                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc314, 0x0042);
1416                                 /* set LED1 to input mode (LED1 and LED2 share same LED) */
1417                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd006, 0x0007);
1418                                 /* set LED2 to RX link_status and activity */
1419                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd007, 0x000A);
1420                                 /* set LED3 to RX link_status */
1421                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd008, 0x0009);
1422
1423                                 /*
1424                                  * reset the res-calibration on t2
1425                                  * serdes; ensures it is stable after
1426                                  * the amcc phy is stable
1427                                  */
1428
1429                                 sds_common_control0  = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
1430                                 sds_common_control0 |= 0x1;
1431                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1432
1433                                 /* release the res-calibration reset */
1434                                 sds_common_control0 &= 0xfffffffe;
1435                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1436
1437                                 i = 0;
1438                                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1439                                                 && (i++ < 5000)) {
1440                                         /* mdelay(1); */
1441                                 }
1442
1443                                 /*
1444                                  * wait for link train done before moving on,
1445                                  * or will get an interupt storm
1446                                  */
1447                                 counter = 0;
1448                                 do {
1449                                         temp_phy_data = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
1450                                                                 (0x200 * (nesdev->mac_index & 1)));
1451                                         if (counter++ > 1000) {
1452                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from link train wait <this is bad, link didnt train!!!>\n");
1453                                                 break;
1454                                         }
1455                                         mdelay(1);
1456                                 } while (((temp_phy_data & 0x0f1f0000) != 0x0f0f0000));
1457                         }
1458                 }
1459         }
1460         return 0;
1461 }
1462
1463
1464 /**
1465  * nes_replenish_nic_rq
1466  */
1467 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1468 {
1469         unsigned long flags;
1470         dma_addr_t bus_address;
1471         struct sk_buff *skb;
1472         struct nes_hw_nic_rq_wqe *nic_rqe;
1473         struct nes_hw_nic *nesnic;
1474         struct nes_device *nesdev;
1475         u32 rx_wqes_posted = 0;
1476
1477         nesnic = &nesvnic->nic;
1478         nesdev = nesvnic->nesdev;
1479         spin_lock_irqsave(&nesnic->rq_lock, flags);
1480         if (nesnic->replenishing_rq !=0) {
1481                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1482                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1483                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1484                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1485                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1486                         add_timer(&nesvnic->rq_wqes_timer);
1487                 } else
1488                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1489                 return;
1490         }
1491         nesnic->replenishing_rq = 1;
1492         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1493         do {
1494                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1495                 if (skb) {
1496                         skb->dev = nesvnic->netdev;
1497
1498                         bus_address = pci_map_single(nesdev->pcidev,
1499                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1500
1501                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1502                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1503                                         cpu_to_le32(nesvnic->max_frame_size);
1504                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1505                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1506                                         cpu_to_le32((u32)bus_address);
1507                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1508                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1509                         nesnic->rx_skb[nesnic->rq_head] = skb;
1510                         nesnic->rq_head++;
1511                         nesnic->rq_head &= nesnic->rq_size - 1;
1512                         atomic_dec(&nesvnic->rx_skbs_needed);
1513                         barrier();
1514                         if (++rx_wqes_posted == 255) {
1515                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1516                                 rx_wqes_posted = 0;
1517                         }
1518                 } else {
1519                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1520                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1521                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1522                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1523                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1524                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1525                                 add_timer(&nesvnic->rq_wqes_timer);
1526                         } else
1527                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1528                         break;
1529                 }
1530         } while (atomic_read(&nesvnic->rx_skbs_needed));
1531         barrier();
1532         if (rx_wqes_posted)
1533                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1534         nesnic->replenishing_rq = 0;
1535 }
1536
1537
1538 /**
1539  * nes_rq_wqes_timeout
1540  */
1541 static void nes_rq_wqes_timeout(unsigned long parm)
1542 {
1543         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1544         printk("%s: Timer fired.\n", __func__);
1545         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1546         if (atomic_read(&nesvnic->rx_skbs_needed))
1547                 nes_replenish_nic_rq(nesvnic);
1548 }
1549
1550
1551 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1552                                void **tcph, u64 *hdr_flags, void *priv)
1553 {
1554         unsigned int ip_len;
1555         struct iphdr *iph;
1556         skb_reset_network_header(skb);
1557         iph = ip_hdr(skb);
1558         if (iph->protocol != IPPROTO_TCP)
1559                 return -1;
1560         ip_len = ip_hdrlen(skb);
1561         skb_set_transport_header(skb, ip_len);
1562         *tcph = tcp_hdr(skb);
1563
1564         *hdr_flags = LRO_IPV4 | LRO_TCP;
1565         *iphdr = iph;
1566         return 0;
1567 }
1568
1569
1570 /**
1571  * nes_init_nic_qp
1572  */
1573 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1574 {
1575         struct nes_hw_cqp_wqe *cqp_wqe;
1576         struct nes_hw_nic_sq_wqe *nic_sqe;
1577         struct nes_hw_nic_qp_context *nic_context;
1578         struct sk_buff *skb;
1579         struct nes_hw_nic_rq_wqe *nic_rqe;
1580         struct nes_vnic *nesvnic = netdev_priv(netdev);
1581         unsigned long flags;
1582         void *vmem;
1583         dma_addr_t pmem;
1584         u64 u64temp;
1585         int ret;
1586         u32 cqp_head;
1587         u32 counter;
1588         u32 wqe_count;
1589         u8 jumbomode=0;
1590
1591         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1592         nesvnic->nic_mem_size = 256 +
1593                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1594                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1595                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1596                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1597                         sizeof(struct nes_hw_nic_qp_context);
1598
1599         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1600                         &nesvnic->nic_pbase);
1601         if (!nesvnic->nic_vbase) {
1602                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1603                 return -ENOMEM;
1604         }
1605         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1606         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1607                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1608
1609         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1610                         ~(unsigned long)(256 - 1));
1611         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1612                         ~(unsigned long long)(256 - 1));
1613
1614         /* Setup the first Fragment buffers */
1615         nesvnic->nic.first_frag_vbase = vmem;
1616
1617         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1618                 nesvnic->nic.frag_paddr[counter] = pmem;
1619                 pmem += sizeof(struct nes_first_frag);
1620         }
1621
1622         /* setup the SQ */
1623         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1624
1625         nesvnic->nic.sq_vbase = (void *)vmem;
1626         nesvnic->nic.sq_pbase = pmem;
1627         nesvnic->nic.sq_head = 0;
1628         nesvnic->nic.sq_tail = 0;
1629         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1630         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1631                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1632                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1633                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1634                                 NES_NIC_SQ_WQE_COMPLETION);
1635                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1636                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1637                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1638                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1639                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1640                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1641         }
1642
1643         nesvnic->get_cqp_request = nes_get_cqp_request;
1644         nesvnic->post_cqp_request = nes_post_cqp_request;
1645         nesvnic->mcrq_mcast_filter = NULL;
1646
1647         spin_lock_init(&nesvnic->nic.sq_lock);
1648         spin_lock_init(&nesvnic->nic.rq_lock);
1649
1650         /* setup the RQ */
1651         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1652         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1653
1654
1655         nesvnic->nic.rq_vbase = vmem;
1656         nesvnic->nic.rq_pbase = pmem;
1657         nesvnic->nic.rq_head = 0;
1658         nesvnic->nic.rq_tail = 0;
1659         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1660
1661         /* setup the CQ */
1662         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1663         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1664
1665         if (nesdev->nesadapter->netdev_count > 2)
1666                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1667         else
1668                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1669
1670         nesvnic->nic_cq.cq_vbase = vmem;
1671         nesvnic->nic_cq.cq_pbase = pmem;
1672         nesvnic->nic_cq.cq_head = 0;
1673         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1674
1675         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1676
1677         /* Send CreateCQ request to CQP */
1678         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1679         cqp_head = nesdev->cqp.sq_head;
1680
1681         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1682         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1683
1684         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1685                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1686                         ((u32)nesvnic->nic_cq.cq_size << 16));
1687         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1688                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1689         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1690         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1691         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1692         u64temp = (unsigned long)&nesvnic->nic_cq;
1693         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1694         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1695                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1696         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1697         if (++cqp_head >= nesdev->cqp.sq_size)
1698                 cqp_head = 0;
1699         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1700         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1701
1702         /* Send CreateQP request to CQP */
1703         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1704         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1705                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1706                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1707         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1708                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1709                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1710         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1711                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1712         }
1713
1714         u64temp = (u64)nesvnic->nic.sq_pbase;
1715         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1716         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1717         u64temp = (u64)nesvnic->nic.rq_pbase;
1718         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1719         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1720
1721         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1722                         NES_CQP_QP_TYPE_NIC);
1723         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1724         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1725                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1726         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1727
1728         if (++cqp_head >= nesdev->cqp.sq_size)
1729                 cqp_head = 0;
1730         nesdev->cqp.sq_head = cqp_head;
1731
1732         barrier();
1733
1734         /* Ring doorbell (2 WQEs) */
1735         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1736
1737         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1738         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1739                         nesvnic->nic.qp_id);
1740
1741         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1742                         NES_EVENT_TIMEOUT);
1743         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1744                         nesvnic->nic.qp_id, ret);
1745         if (!ret) {
1746                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1747                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1748                                 nesvnic->nic_pbase);
1749                 return -EIO;
1750         }
1751
1752         /* Populate the RQ */
1753         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1754                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1755                 if (!skb) {
1756                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1757
1758                         nes_destroy_nic_qp(nesvnic);
1759                         return -ENOMEM;
1760                 }
1761
1762                 skb->dev = netdev;
1763
1764                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1765                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1766
1767                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1768                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1769                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1770                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1771                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1772                 nesvnic->nic.rx_skb[counter] = skb;
1773         }
1774
1775         wqe_count = NES_NIC_WQ_SIZE - 1;
1776         nesvnic->nic.rq_head = wqe_count;
1777         barrier();
1778         do {
1779                 counter = min(wqe_count, ((u32)255));
1780                 wqe_count -= counter;
1781                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1782         } while (wqe_count);
1783         init_timer(&nesvnic->rq_wqes_timer);
1784         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1785         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1786         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1787         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1788         {
1789                 nes_nic_init_timer(nesdev);
1790                 if (netdev->mtu > 1500)
1791                         jumbomode = 1;
1792                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1793         }
1794         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1795         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1796         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1797         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1798         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1799         nesvnic->lro_mgr.dev            = netdev;
1800         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1801         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1802         return 0;
1803 }
1804
1805
1806 /**
1807  * nes_destroy_nic_qp
1808  */
1809 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1810 {
1811         u64 u64temp;
1812         dma_addr_t bus_address;
1813         struct nes_device *nesdev = nesvnic->nesdev;
1814         struct nes_hw_cqp_wqe *cqp_wqe;
1815         struct nes_hw_nic_sq_wqe *nic_sqe;
1816         struct nes_hw_nic_rq_wqe *nic_rqe;
1817         __le16 *wqe_fragment_length;
1818         u16  wqe_fragment_index;
1819         u64 wqe_frag;
1820         u32 cqp_head;
1821         unsigned long flags;
1822         int ret;
1823
1824         /* Free remaining NIC receive buffers */
1825         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1826                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1827                 wqe_frag  = (u64)le32_to_cpu(
1828                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1829                 wqe_frag |= ((u64)le32_to_cpu(
1830                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1831                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1832                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1833                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1834                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1835         }
1836
1837         /* Free remaining NIC transmit buffers */
1838         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1839                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1840                 wqe_fragment_index = 1;
1841                 wqe_fragment_length = (__le16 *)
1842                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1843                 /* bump past the vlan tag */
1844                 wqe_fragment_length++;
1845                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1846                         u64temp = (u64)le32_to_cpu(
1847                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1848                                 wqe_fragment_index*2]);
1849                         u64temp += ((u64)le32_to_cpu(
1850                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1851                                 + wqe_fragment_index*2]))<<32;
1852                         bus_address = (dma_addr_t)u64temp;
1853                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1854                                         nesvnic->nic.first_frag_overflow)) {
1855                                 pci_unmap_single(nesdev->pcidev,
1856                                                 bus_address,
1857                                                 le16_to_cpu(wqe_fragment_length[
1858                                                         wqe_fragment_index++]),
1859                                                 PCI_DMA_TODEVICE);
1860                         }
1861                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1862                                 if (wqe_fragment_length[wqe_fragment_index]) {
1863                                         u64temp = le32_to_cpu(
1864                                                 nic_sqe->wqe_words[
1865                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1866                                                 wqe_fragment_index*2]);
1867                                         u64temp += ((u64)le32_to_cpu(
1868                                                 nic_sqe->wqe_words[
1869                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1870                                                 wqe_fragment_index*2]))<<32;
1871                                         bus_address = (dma_addr_t)u64temp;
1872                                         pci_unmap_page(nesdev->pcidev,
1873                                                         bus_address,
1874                                                         le16_to_cpu(
1875                                                         wqe_fragment_length[
1876                                                         wqe_fragment_index]),
1877                                                         PCI_DMA_TODEVICE);
1878                                 } else
1879                                         break;
1880                         }
1881                 }
1882                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1883                         dev_kfree_skb(
1884                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1885
1886                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1887                                         & (nesvnic->nic.sq_size - 1);
1888         }
1889
1890         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1891
1892         /* Destroy NIC QP */
1893         cqp_head = nesdev->cqp.sq_head;
1894         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1895         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1896
1897         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1898                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1899         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1900                 nesvnic->nic.qp_id);
1901
1902         if (++cqp_head >= nesdev->cqp.sq_size)
1903                 cqp_head = 0;
1904
1905         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1906
1907         /* Destroy NIC CQ */
1908         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1909         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1910                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1911         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1912                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1913
1914         if (++cqp_head >= nesdev->cqp.sq_size)
1915                 cqp_head = 0;
1916
1917         nesdev->cqp.sq_head = cqp_head;
1918         barrier();
1919
1920         /* Ring doorbell (2 WQEs) */
1921         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1922
1923         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1924         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1925                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1926                         cqp_head, nesdev->cqp.sq_head,
1927                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1928
1929         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1930                         NES_EVENT_TIMEOUT);
1931
1932         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1933                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1934                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1935         if (!ret) {
1936                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1937                                 nesvnic->nic.qp_id);
1938         }
1939
1940         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1941                         nesvnic->nic_pbase);
1942 }
1943
1944 /**
1945  * nes_napi_isr
1946  */
1947 int nes_napi_isr(struct nes_device *nesdev)
1948 {
1949         struct nes_adapter *nesadapter = nesdev->nesadapter;
1950         u32 int_stat;
1951
1952         if (nesdev->napi_isr_ran) {
1953                 /* interrupt status has already been read in ISR */
1954                 int_stat = nesdev->int_stat;
1955         } else {
1956                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1957                 nesdev->int_stat = int_stat;
1958                 nesdev->napi_isr_ran = 1;
1959         }
1960
1961         int_stat &= nesdev->int_req;
1962         /* iff NIC, process here, else wait for DPC */
1963         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1964                 nesdev->napi_isr_ran = 0;
1965                 nes_write32(nesdev->regs + NES_INT_STAT,
1966                         (int_stat &
1967                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1968
1969                 /* Process the CEQs */
1970                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1971
1972                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1973                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1974                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1975                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1976                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1977                                 /* Enable Periodic timer interrupts */
1978                                 nesdev->int_req |= NES_INT_TIMER;
1979                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1980                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1981                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1982                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1983                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1984                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1985                         }
1986
1987                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1988                         {
1989                                 nes_nic_init_timer(nesdev);
1990                         }
1991                         /* Enable interrupts, except CEQs */
1992                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1993                 } else {
1994                         /* Enable interrupts, make sure timer is off */
1995                         nesdev->int_req &= ~NES_INT_TIMER;
1996                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1997                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1998                 }
1999                 nesdev->deepcq_count = 0;
2000                 return 1;
2001         } else {
2002                 return 0;
2003         }
2004 }
2005
2006 static void process_critical_error(struct nes_device *nesdev)
2007 {
2008         u32 debug_error;
2009         u32 nes_idx_debug_error_masks0 = 0;
2010         u16 error_module = 0;
2011
2012         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2013         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2014                         (u16)debug_error);
2015         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2016                         0x01010000 | (debug_error & 0x0000ffff));
2017         if (crit_err_count++ > 10)
2018                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2019         error_module = (u16) (debug_error & 0x1F00) >> 8;
2020         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2021                         nes_max_critical_error_count) {
2022                 printk(KERN_ERR PFX "Masking off critical error for module "
2023                         "0x%02X\n", (u16)error_module);
2024                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2025                         NES_IDX_DEBUG_ERROR_MASKS0);
2026                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2027                         nes_idx_debug_error_masks0 | (1 << error_module));
2028         }
2029 }
2030 /**
2031  * nes_dpc
2032  */
2033 void nes_dpc(unsigned long param)
2034 {
2035         struct nes_device *nesdev = (struct nes_device *)param;
2036         struct nes_adapter *nesadapter = nesdev->nesadapter;
2037         u32 counter;
2038         u32 loop_counter = 0;
2039         u32 int_status_bit;
2040         u32 int_stat;
2041         u32 timer_stat;
2042         u32 temp_int_stat;
2043         u32 intf_int_stat;
2044         u32 processed_intf_int = 0;
2045         u16 processed_timer_int = 0;
2046         u16 completion_ints = 0;
2047         u16 timer_ints = 0;
2048
2049         /* nes_debug(NES_DBG_ISR, "\n"); */
2050
2051         do {
2052                 timer_stat = 0;
2053                 if (nesdev->napi_isr_ran) {
2054                         nesdev->napi_isr_ran = 0;
2055                         int_stat = nesdev->int_stat;
2056                 } else
2057                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2058                 if (processed_intf_int != 0)
2059                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2060                 else
2061                         int_stat &= nesdev->int_req;
2062                 if (processed_timer_int == 0) {
2063                         processed_timer_int = 1;
2064                         if (int_stat & NES_INT_TIMER) {
2065                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2066                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2067                                         int_stat &= ~NES_INT_TIMER;
2068                                 }
2069                         }
2070                 } else {
2071                         int_stat &= ~NES_INT_TIMER;
2072                 }
2073
2074                 if (int_stat) {
2075                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2076                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2077                                 /* Ack the interrupts */
2078                                 nes_write32(nesdev->regs+NES_INT_STAT,
2079                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2080                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2081                         }
2082
2083                         temp_int_stat = int_stat;
2084                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2085                                 if (int_stat & int_status_bit) {
2086                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2087                                         temp_int_stat &= ~int_status_bit;
2088                                         completion_ints = 1;
2089                                 }
2090                                 if (!(temp_int_stat & 0x0000ffff))
2091                                         break;
2092                                 int_status_bit <<= 1;
2093                         }
2094
2095                         /* Process the AEQ for this pci function */
2096                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2097                         if (int_stat & int_status_bit) {
2098                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2099                         }
2100
2101                         /* Process the MAC interrupt for this pci function */
2102                         int_status_bit = 1 << (24 + nesdev->mac_index);
2103                         if (int_stat & int_status_bit) {
2104                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2105                         }
2106
2107                         if (int_stat & NES_INT_TIMER) {
2108                                 if (timer_stat & nesdev->timer_int_req) {
2109                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2110                                                         (timer_stat & nesdev->timer_int_req) |
2111                                                         ~(nesdev->nesadapter->timer_int_req));
2112                                         timer_ints = 1;
2113                                 }
2114                         }
2115
2116                         if (int_stat & NES_INT_INTF) {
2117                                 processed_intf_int = 1;
2118                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2119                                 intf_int_stat &= nesdev->intf_int_req;
2120                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2121                                         process_critical_error(nesdev);
2122                                 }
2123                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2124                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2125                                         BUG();
2126                                 }
2127                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2128                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2129                                         BUG();
2130                                 }
2131                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2132                         }
2133
2134                         if (int_stat & NES_INT_TSW) {
2135                         }
2136                 }
2137                 /* Don't use the interface interrupt bit stay in loop */
2138                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2139                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2140         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2141
2142         if (timer_ints == 1) {
2143                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2144                         if (completion_ints == 0) {
2145                                 nesdev->timer_only_int_count++;
2146                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2147                                         nesdev->timer_only_int_count = 0;
2148                                         nesdev->int_req &= ~NES_INT_TIMER;
2149                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2150                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2151                                 } else {
2152                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2153                                 }
2154                         } else {
2155                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2156                                 {
2157                                         nes_nic_init_timer(nesdev);
2158                                 }
2159                                 nesdev->timer_only_int_count = 0;
2160                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2161                         }
2162                 } else {
2163                         nesdev->timer_only_int_count = 0;
2164                         nesdev->int_req &= ~NES_INT_TIMER;
2165                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2166                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2167                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2168                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2169                 }
2170         } else {
2171                 if ( (completion_ints == 1) &&
2172                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2173                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2174                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2175                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2176                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2177                         nesdev->timer_only_int_count = 0;
2178                         nesdev->int_req |= NES_INT_TIMER;
2179                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2180                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2181                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2182                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2183                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2184                 } else {
2185                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2186                 }
2187         }
2188         nesdev->deepcq_count = 0;
2189 }
2190
2191
2192 /**
2193  * nes_process_ceq
2194  */
2195 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2196 {
2197         u64 u64temp;
2198         struct nes_hw_cq *cq;
2199         u32 head;
2200         u32 ceq_size;
2201
2202         /* nes_debug(NES_DBG_CQ, "\n"); */
2203         head = ceq->ceq_head;
2204         ceq_size = ceq->ceq_size;
2205
2206         do {
2207                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2208                                 NES_CEQE_VALID) {
2209                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2210                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2211                         u64temp <<= 1;
2212                         cq = *((struct nes_hw_cq **)&u64temp);
2213                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2214                         barrier();
2215                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2216
2217                         /* call the event handler */
2218                         cq->ce_handler(nesdev, cq);
2219
2220                         if (++head >= ceq_size)
2221                                 head = 0;
2222                 } else {
2223                         break;
2224                 }
2225
2226         } while (1);
2227
2228         ceq->ceq_head = head;
2229 }
2230
2231
2232 /**
2233  * nes_process_aeq
2234  */
2235 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2236 {
2237         /* u64 u64temp; */
2238         u32 head;
2239         u32 aeq_size;
2240         u32 aeqe_misc;
2241         u32 aeqe_cq_id;
2242         struct nes_hw_aeqe volatile *aeqe;
2243
2244         head = aeq->aeq_head;
2245         aeq_size = aeq->aeq_size;
2246
2247         do {
2248                 aeqe = &aeq->aeq_vbase[head];
2249                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2250                         break;
2251                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2252                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2253                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2254                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2255                                 /* dealing with an accelerated QP related AE */
2256                                 /*
2257                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2258                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2259                                  */
2260                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2261                         } else {
2262                                 /* TODO: dealing with a CQP related AE */
2263                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2264                                                 (u16)(aeqe_misc >> 16));
2265                         }
2266                 }
2267
2268                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2269
2270                 if (++head >= aeq_size)
2271                         head = 0;
2272         }
2273         while (1);
2274         aeq->aeq_head = head;
2275 }
2276
2277 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2278 {
2279         struct nes_adapter *nesadapter = nesdev->nesadapter;
2280         u32 reset_value;
2281         u32 i=0;
2282         u32 u32temp;
2283
2284         if (nesadapter->hw_rev == NE020_REV) {
2285                 return;
2286         }
2287         mh_detected++;
2288
2289         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2290
2291         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2292                 reset_value |= 0x0000001d;
2293         else
2294                 reset_value |= 0x0000002d;
2295
2296         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2297                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2298                         nesadapter->link_interrupt_count[0] = 0;
2299                         nesadapter->link_interrupt_count[1] = 0;
2300                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2301                         if (0x00000040 & u32temp)
2302                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2303                         else
2304                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2305
2306                         reset_value |= 0x0000003d;
2307                 }
2308                 nesadapter->link_interrupt_count[mac_index] = 0;
2309         }
2310
2311         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2312
2313         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2314                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2315
2316         if (0x0000003d == (reset_value & 0x0000003d)) {
2317                 u32 pcs_control_status0, pcs_control_status1;
2318
2319                 for (i = 0; i < 10; i++) {
2320                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2321                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2322                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2323                              && (pcs_control_status0 & 0x00100000))
2324                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2325                                 && (pcs_control_status1 & 0x00100000)))
2326                                 continue;
2327                         else
2328                                 break;
2329                 }
2330                 if (10 == i) {
2331                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2332                         if (0x00000040 & u32temp)
2333                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2334                         else
2335                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2336
2337                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2338
2339                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2340                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2341                 }
2342         }
2343 }
2344
2345 /**
2346  * nes_process_mac_intr
2347  */
2348 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2349 {
2350         unsigned long flags;
2351         u32 pcs_control_status;
2352         struct nes_adapter *nesadapter = nesdev->nesadapter;
2353         struct nes_vnic *nesvnic;
2354         u32 mac_status;
2355         u32 mac_index = nesdev->mac_index;
2356         u32 u32temp;
2357         u16 phy_data;
2358         u16 temp_phy_data;
2359         u32 pcs_val  = 0x0f0f0000;
2360         u32 pcs_mask = 0x0f1f0000;
2361
2362         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2363         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2364                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2365                 return;
2366         }
2367         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2368         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2369
2370         /* ack the MAC interrupt */
2371         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2372         /* Clear the interrupt */
2373         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2374
2375         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2376
2377         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2378                 nesdev->link_status_interrupts++;
2379                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2380                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2381                         nes_reset_link(nesdev, mac_index);
2382                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2383                 }
2384                 /* read the PHY interrupt status register */
2385                 if ((nesadapter->OneG_Mode) &&
2386                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2387                         do {
2388                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2389                                                 nesadapter->phy_index[mac_index], &phy_data);
2390                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2391                                                 nesadapter->phy_index[mac_index], phy_data);
2392                         } while (phy_data&0x8000);
2393
2394                         temp_phy_data = 0;
2395                         do {
2396                                 nes_read_1G_phy_reg(nesdev, 0x11,
2397                                                 nesadapter->phy_index[mac_index], &phy_data);
2398                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2399                                                 nesadapter->phy_index[mac_index], phy_data);
2400                                 if (temp_phy_data == phy_data)
2401                                         break;
2402                                 temp_phy_data = phy_data;
2403                         } while (1);
2404
2405                         nes_read_1G_phy_reg(nesdev, 0x1e,
2406                                         nesadapter->phy_index[mac_index], &phy_data);
2407                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2408                                         nesadapter->phy_index[mac_index], phy_data);
2409
2410                         nes_read_1G_phy_reg(nesdev, 1,
2411                                         nesadapter->phy_index[mac_index], &phy_data);
2412                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2413                                         nesadapter->phy_index[mac_index], phy_data);
2414
2415                         if (temp_phy_data & 0x1000) {
2416                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2417                                 phy_data = 4;
2418                         } else {
2419                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2420                         }
2421                 }
2422                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2423                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2424                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2425
2426                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2427                         switch (mac_index) {
2428                         case 1:
2429                         case 3:
2430                                 pcs_control_status = nes_read_indexed(nesdev,
2431                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2432                                 break;
2433                         default:
2434                                 pcs_control_status = nes_read_indexed(nesdev,
2435                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2436                                 break;
2437                         }
2438                 } else {
2439                         pcs_control_status = nes_read_indexed(nesdev,
2440                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2441                         pcs_control_status = nes_read_indexed(nesdev,
2442                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2443                 }
2444
2445                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2446                                 mac_index, pcs_control_status);
2447                 if ((nesadapter->OneG_Mode) &&
2448                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2449                         u32temp = 0x01010000;
2450                         if (nesadapter->port_count > 2) {
2451                                 u32temp |= 0x02020000;
2452                         }
2453                         if ((pcs_control_status & u32temp)!= u32temp) {
2454                                 phy_data = 0;
2455                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2456                         }
2457                 } else {
2458                         switch (nesadapter->phy_type[mac_index]) {
2459                         case NES_PHY_TYPE_IRIS:
2460                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2461                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2462                                 u32temp = 20;
2463                                 do {
2464                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2465                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2466                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2467                                                 break;
2468                                         temp_phy_data = phy_data;
2469                                 } while (1);
2470                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2471                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2472                                 break;
2473
2474                         case NES_PHY_TYPE_ARGUS:
2475                                 /* clear the alarms */
2476                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2477                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2478                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2479                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2480                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2481                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2482                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2483                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2484                                 /* check link status */
2485                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2486                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2487                                 u32temp = 100;
2488                                 do {
2489                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2490
2491                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2492                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2493                                                 break;
2494                                         temp_phy_data = phy_data;
2495                                 } while (1);
2496                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2497                                         __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2498                                 break;
2499
2500                         case NES_PHY_TYPE_PUMA_1G:
2501                                 if (mac_index < 2)
2502                                         pcs_val = pcs_mask = 0x01010000;
2503                                 else
2504                                         pcs_val = pcs_mask = 0x02020000;
2505                                 /* fall through */
2506                         default:
2507                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2508                                 break;
2509                         }
2510                 }
2511
2512                 if (phy_data & 0x0004) {
2513                         nesadapter->mac_link_down[mac_index] = 0;
2514                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2515                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2516                                                 nesvnic->linkup);
2517                                 if (nesvnic->linkup == 0) {
2518                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2519                                                         nesvnic->netdev->name, nesvnic->netdev);
2520                                         if (netif_queue_stopped(nesvnic->netdev))
2521                                                 netif_start_queue(nesvnic->netdev);
2522                                         nesvnic->linkup = 1;
2523                                         netif_carrier_on(nesvnic->netdev);
2524                                 }
2525                         }
2526                 } else {
2527                         nesadapter->mac_link_down[mac_index] = 1;
2528                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2529                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2530                                                 nesvnic->linkup);
2531                                 if (nesvnic->linkup == 1) {
2532                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2533                                                         nesvnic->netdev->name, nesvnic->netdev);
2534                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2535                                                 netif_stop_queue(nesvnic->netdev);
2536                                         nesvnic->linkup = 0;
2537                                         netif_carrier_off(nesvnic->netdev);
2538                                 }
2539                         }
2540                 }
2541         }
2542
2543         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2544 }
2545
2546
2547
2548 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2549 {
2550         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2551
2552         netif_rx_schedule(&nesvnic->napi);
2553 }
2554
2555
2556 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2557 * getting out of nic_ce_handler
2558 */
2559 #define MAX_RQES_TO_PROCESS     384
2560
2561 /**
2562  * nes_nic_ce_handler
2563  */
2564 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2565 {
2566         u64 u64temp;
2567         dma_addr_t bus_address;
2568         struct nes_hw_nic *nesnic;
2569         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2570         struct nes_adapter *nesadapter = nesdev->nesadapter;
2571         struct nes_hw_nic_rq_wqe *nic_rqe;
2572         struct nes_hw_nic_sq_wqe *nic_sqe;
2573         struct sk_buff *skb;
2574         struct sk_buff *rx_skb;
2575         __le16 *wqe_fragment_length;
2576         u32 head;
2577         u32 cq_size;
2578         u32 rx_pkt_size;
2579         u32 cqe_count=0;
2580         u32 cqe_errv;
2581         u32 cqe_misc;
2582         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2583         u16 vlan_tag;
2584         u16 pkt_type;
2585         u16 rqes_processed = 0;
2586         u8 sq_cqes = 0;
2587         u8 nes_use_lro = 0;
2588
2589         head = cq->cq_head;
2590         cq_size = cq->cq_size;
2591         cq->cqes_pending = 1;
2592         if (nesvnic->netdev->features & NETIF_F_LRO)
2593                 nes_use_lro = 1;
2594         do {
2595                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2596                                 NES_NIC_CQE_VALID) {
2597                         nesnic = &nesvnic->nic;
2598                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2599                         if (cqe_misc & NES_NIC_CQE_SQ) {
2600                                 sq_cqes++;
2601                                 wqe_fragment_index = 1;
2602                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2603                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2604                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2605                                 /* bump past the vlan tag */
2606                                 wqe_fragment_length++;
2607                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2608                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2609                                                         wqe_fragment_index * 2]);
2610                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2611                                                         wqe_fragment_index * 2])) << 32;
2612                                         bus_address = (dma_addr_t)u64temp;
2613                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2614                                                 pci_unmap_single(nesdev->pcidev,
2615                                                                 bus_address,
2616                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2617                                                                 PCI_DMA_TODEVICE);
2618                                         }
2619                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2620                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2621                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2622                                                                                 wqe_fragment_index * 2]);
2623                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2624                                                                                 + wqe_fragment_index * 2])) <<32;
2625                                                         bus_address = (dma_addr_t)u64temp;
2626                                                         pci_unmap_page(nesdev->pcidev,
2627                                                                         bus_address,
2628                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2629                                                                         PCI_DMA_TODEVICE);
2630                                                 } else
2631                                                         break;
2632                                         }
2633                                         if (skb)
2634                                                 dev_kfree_skb_any(skb);
2635                                 }
2636                                 nesnic->sq_tail++;
2637                                 nesnic->sq_tail &= nesnic->sq_size-1;
2638                                 if (sq_cqes > 128) {
2639                                         barrier();
2640                                 /* restart the queue if it had been stopped */
2641                                 if (netif_queue_stopped(nesvnic->netdev))
2642                                         netif_wake_queue(nesvnic->netdev);
2643                                         sq_cqes = 0;
2644                                 }
2645                         } else {
2646                                 rqes_processed ++;
2647
2648                                 cq->rx_cqes_completed++;
2649                                 cq->rx_pkts_indicated++;
2650                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2651                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2652                                 /* Get the skb */
2653                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2654                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2655                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2656                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2657                                 pci_unmap_single(nesdev->pcidev, bus_address,
2658                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2659                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2660                                 /* rx_skb->len = rx_pkt_size; */
2661                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2662                                 skb_put(rx_skb, rx_pkt_size);
2663                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2664                                 nesnic->rq_tail++;
2665                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2666
2667                                 atomic_inc(&nesvnic->rx_skbs_needed);
2668                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2669                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2670                                                         cq->cq_number | (cqe_count << 16));
2671                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2672                                         nesdev->currcq_count += cqe_count;
2673                                         cqe_count = 0;
2674                                         nes_replenish_nic_rq(nesvnic);
2675                                 }
2676                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2677                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2678                                 rx_skb->ip_summed = CHECKSUM_NONE;
2679
2680                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2681                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2682                                         if ((cqe_errv &
2683                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2684                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2685                                                 if (nesvnic->rx_checksum_disabled == 0) {
2686                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2687                                                 }
2688                                         } else
2689                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2690                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2691                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2692
2693                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2694                                         if ((cqe_errv &
2695                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2696                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2697                                                 if (nesvnic->rx_checksum_disabled == 0) {
2698                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2699                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2700                                                                   nesvnic->netdev->name); */
2701                                                 }
2702                                         } else
2703                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2704                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2705                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2706                                         }
2707                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2708                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2709
2710                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2711                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2712                                                 rx_skb = NULL;
2713                                 }
2714                                 if (rx_skb == NULL)
2715                                         goto skip_rx_indicate0;
2716
2717
2718                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2719                                     (nesvnic->vlan_grp != NULL)) {
2720                                         vlan_tag = (u16)(le32_to_cpu(
2721                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2722                                                         >> 16);
2723                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2724                                                         nesvnic->netdev->name, vlan_tag);
2725                                         if (nes_use_lro)
2726                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2727                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2728                                         else
2729                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2730                                 } else {
2731                                         if (nes_use_lro)
2732                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2733                                         else
2734                                                 nes_netif_rx(rx_skb);
2735                                 }
2736
2737 skip_rx_indicate0:
2738                                 nesvnic->netdev->last_rx = jiffies;
2739                                 /* nesvnic->netstats.rx_packets++; */
2740                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2741                         }
2742
2743                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2744                         /* Accounting... */
2745                         cqe_count++;
2746                         if (++head >= cq_size)
2747                                 head = 0;
2748                         if (cqe_count == 255) {
2749                                 /* Replenish Nic CQ */
2750                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2751                                                 cq->cq_number | (cqe_count << 16));
2752                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2753                                 nesdev->currcq_count += cqe_count;
2754                                 cqe_count = 0;
2755                         }
2756
2757                         if (cq->rx_cqes_completed >= nesvnic->budget)
2758                                 break;
2759                 } else {
2760                         cq->cqes_pending = 0;
2761                         break;
2762                 }
2763
2764         } while (1);
2765
2766         if (nes_use_lro)
2767                 lro_flush_all(&nesvnic->lro_mgr);
2768         if (sq_cqes) {
2769                 barrier();
2770                 /* restart the queue if it had been stopped */
2771                 if (netif_queue_stopped(nesvnic->netdev))
2772                         netif_wake_queue(nesvnic->netdev);
2773         }
2774         cq->cq_head = head;
2775         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2776                         cq->cq_number, cqe_count, cq->cq_head); */
2777         cq->cqe_allocs_pending = cqe_count;
2778         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2779         {
2780                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2781                 nesdev->currcq_count += cqe_count;
2782                 nes_nic_tune_timer(nesdev);
2783         }
2784         if (atomic_read(&nesvnic->rx_skbs_needed))
2785                 nes_replenish_nic_rq(nesvnic);
2786 }
2787
2788
2789 /**
2790  * nes_cqp_ce_handler
2791  */
2792 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2793 {
2794         u64 u64temp;
2795         unsigned long flags;
2796         struct nes_hw_cqp *cqp = NULL;
2797         struct nes_cqp_request *cqp_request;
2798         struct nes_hw_cqp_wqe *cqp_wqe;
2799         u32 head;
2800         u32 cq_size;
2801         u32 cqe_count=0;
2802         u32 error_code;
2803         /* u32 counter; */
2804
2805         head = cq->cq_head;
2806         cq_size = cq->cq_size;
2807
2808         do {
2809                 /* process the CQE */
2810                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2811                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2812
2813                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2814                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2815                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2816                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2817                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2818                         cqp = *((struct nes_hw_cqp **)&u64temp);
2819
2820                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2821                         if (error_code) {
2822                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2823                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2824                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2825                                                 (u16)(error_code >> 16),
2826                                                 (u16)error_code);
2827                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2828                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2829                         }
2830
2831                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2832                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2833                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2834                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2835                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2836                         if (cqp_request) {
2837                                 if (cqp_request->waiting) {
2838                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2839                                         cqp_request->major_code = (u16)(error_code >> 16);
2840                                         cqp_request->minor_code = (u16)error_code;
2841                                         barrier();
2842                                         cqp_request->request_done = 1;
2843                                         wake_up(&cqp_request->waitq);
2844                                         nes_put_cqp_request(nesdev, cqp_request);
2845                                 } else {
2846                                         if (cqp_request->callback)
2847                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2848                                         nes_free_cqp_request(nesdev, cqp_request);
2849                                 }
2850                         } else {
2851                                 wake_up(&nesdev->cqp.waitq);
2852                         }
2853
2854                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2855                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2856                         if (++cqp->sq_tail >= cqp->sq_size)
2857                                 cqp->sq_tail = 0;
2858
2859                         /* Accounting... */
2860                         cqe_count++;
2861                         if (++head >= cq_size)
2862                                 head = 0;
2863                 } else {
2864                         break;
2865                 }
2866         } while (1);
2867         cq->cq_head = head;
2868
2869         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2870         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2871                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2872                         (nesdev->cqp.sq_size - 1)) != 1)) {
2873                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2874                                 struct nes_cqp_request, list);
2875                 list_del_init(&cqp_request->list);
2876                 head = nesdev->cqp.sq_head++;
2877                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2878                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2879                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2880                 barrier();
2881                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2882                         cpu_to_le32((u32)((unsigned long)cqp_request));
2883                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2884                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2885                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2886                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2887                 /* Ring doorbell (1 WQEs) */
2888                 barrier();
2889                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2890         }
2891         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2892
2893         /* Arm the CCQ */
2894         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2895                         cq->cq_number);
2896         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2897 }
2898
2899
2900 /**
2901  * nes_process_iwarp_aeqe
2902  */
2903 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2904                                    struct nes_hw_aeqe *aeqe)
2905 {
2906         u64 context;
2907         u64 aeqe_context = 0;
2908         unsigned long flags;
2909         struct nes_qp *nesqp;
2910         int resource_allocated;
2911         /* struct iw_cm_id *cm_id; */
2912         struct nes_adapter *nesadapter = nesdev->nesadapter;
2913         struct ib_event ibevent;
2914         /* struct iw_cm_event cm_event; */
2915         u32 aeq_info;
2916         u32 next_iwarp_state = 0;
2917         u16 async_event_id;
2918         u8 tcp_state;
2919         u8 iwarp_state;
2920
2921         nes_debug(NES_DBG_AEQ, "\n");
2922         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2923         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2924                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2925                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2926         } else {
2927                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2928                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2929                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2930                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2931                 BUG_ON(!context);
2932         }
2933
2934         async_event_id = (u16)aeq_info;
2935         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2936         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2937         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2938                         " Tcp state = %s, iWARP state = %s\n",
2939                         async_event_id,
2940                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2941                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2942
2943         switch (async_event_id) {
2944                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2945                         nesqp = *((struct nes_qp **)&context);
2946                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2947                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2948                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2949                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2950                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2951                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2952                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2953                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2954                                                 async_event_id, nesqp->last_aeq, tcp_state);
2955                         }
2956                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2957                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2958                                 /* FIN Received but tcp state or IB state moved on,
2959                                                 should expect a close complete */
2960                                 return;
2961                         }
2962                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2963                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2964                 case NES_AEQE_AEID_TERMINATE_SENT:
2965                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2966                 case NES_AEQE_AEID_RESET_SENT:
2967                         nesqp = *((struct nes_qp **)&context);
2968                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2969                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2970                         }
2971                         spin_lock_irqsave(&nesqp->lock, flags);
2972                         nesqp->hw_iwarp_state = iwarp_state;
2973                         nesqp->hw_tcp_state = tcp_state;
2974                         nesqp->last_aeq = async_event_id;
2975
2976                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2977                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2978                                 nesqp->hte_added = 0;
2979                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2980                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2981                                                 nesqp->hwqp.qp_id);
2982                                 nes_hw_modify_qp(nesdev, nesqp,
2983                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2984                                 spin_lock_irqsave(&nesqp->lock, flags);
2985                         }
2986
2987                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2988                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2989                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2990                                 switch (nesqp->hw_iwarp_state) {
2991                                         case NES_AEQE_IWARP_STATE_RTS:
2992                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2993                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2994                                                 break;
2995                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2996                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2997                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2998                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2999                                                         next_iwarp_state |= 0x02000000;
3000                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3001                                                 }
3002                                                 break;
3003                                         default:
3004                                                 next_iwarp_state = 0;
3005                                 }
3006                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3007                                 if (next_iwarp_state) {
3008                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3009                                                         " also added another reference\n",
3010                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3011                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3012                                 }
3013                                 nes_cm_disconn(nesqp);
3014                         } else {
3015                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3016                                         /* FIN Received but ib state not RTS,
3017                                                         close complete will be on its way */
3018                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3019                                         return;
3020                                 }
3021                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3022                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3023                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
3024                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3025                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3026                                                         " also added another reference\n",
3027                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3028                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3029                                 }
3030                                 nes_cm_disconn(nesqp);
3031                         }
3032                         break;
3033                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3034                         nesqp = *((struct nes_qp **)&context);
3035                         spin_lock_irqsave(&nesqp->lock, flags);
3036                         nesqp->hw_iwarp_state = iwarp_state;
3037                         nesqp->hw_tcp_state = tcp_state;
3038                         nesqp->last_aeq = async_event_id;
3039                         spin_unlock_irqrestore(&nesqp->lock, flags);
3040                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
3041                                         " event on QP%u \n  Q2 Data:\n",
3042                                         nesqp->hwqp.qp_id);
3043                         if (nesqp->ibqp.event_handler) {
3044                                 ibevent.device = nesqp->ibqp.device;
3045                                 ibevent.element.qp = &nesqp->ibqp;
3046                                 ibevent.event = IB_EVENT_QP_FATAL;
3047                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3048                         }
3049                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3050                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
3051                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3052                                 nes_cm_disconn(nesqp);
3053                         } else {
3054                                 nesqp->in_disconnect = 0;
3055                                 wake_up(&nesqp->kick_waitq);
3056                         }
3057                         break;
3058                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3059                         nesqp = *((struct nes_qp **)&context);
3060                         spin_lock_irqsave(&nesqp->lock, flags);
3061                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3062                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3063                         nesqp->last_aeq = async_event_id;
3064                         if (nesqp->cm_id) {
3065                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3066                                                 " event on QP%u, remote IP = 0x%08X \n",
3067                                                 nesqp->hwqp.qp_id,
3068                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
3069                         } else {
3070                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3071                                                 " event on QP%u \n",
3072                                                 nesqp->hwqp.qp_id);
3073                         }
3074                         spin_unlock_irqrestore(&nesqp->lock, flags);
3075                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
3076                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3077                         if (nesqp->ibqp.event_handler) {
3078                                 ibevent.device = nesqp->ibqp.device;
3079                                 ibevent.element.qp = &nesqp->ibqp;
3080                                 ibevent.event = IB_EVENT_QP_FATAL;
3081                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3082                         }
3083                         break;
3084                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3085                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
3086                                 nesqp = nesadapter->qp_table[le32_to_cpu(
3087                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3088                         } else {
3089                                 /* TODO: get the actual WQE and mask off wqe index */
3090                                 context &= ~((u64)511);
3091                                 nesqp = *((struct nes_qp **)&context);
3092                         }
3093                         spin_lock_irqsave(&nesqp->lock, flags);
3094                         nesqp->hw_iwarp_state = iwarp_state;
3095                         nesqp->hw_tcp_state = tcp_state;
3096                         nesqp->last_aeq = async_event_id;
3097                         spin_unlock_irqrestore(&nesqp->lock, flags);
3098                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3099                                         nesqp->hwqp.qp_id);
3100                         if (nesqp->ibqp.event_handler) {
3101                                 ibevent.device = nesqp->ibqp.device;
3102                                 ibevent.element.qp = &nesqp->ibqp;
3103                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3104                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3105                         }
3106                         break;
3107                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3108                         nesqp = *((struct nes_qp **)&context);
3109                         spin_lock_irqsave(&nesqp->lock, flags);
3110                         nesqp->hw_iwarp_state = iwarp_state;
3111                         nesqp->hw_tcp_state = tcp_state;
3112                         nesqp->last_aeq = async_event_id;
3113                         spin_unlock_irqrestore(&nesqp->lock, flags);
3114                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3115                                         nesqp->hwqp.qp_id);
3116                         if (nesqp->ibqp.event_handler) {
3117                                 ibevent.device = nesqp->ibqp.device;
3118                                 ibevent.element.qp = &nesqp->ibqp;
3119                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3120                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3121                         }
3122                         break;
3123                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3124                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3125                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3126                         spin_lock_irqsave(&nesqp->lock, flags);
3127                         nesqp->hw_iwarp_state = iwarp_state;
3128                         nesqp->hw_tcp_state = tcp_state;
3129                         nesqp->last_aeq = async_event_id;
3130                         spin_unlock_irqrestore(&nesqp->lock, flags);
3131                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3132                                         " nesqp = %p, AE reported %p\n",
3133                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3134                         if (nesqp->ibqp.event_handler) {
3135                                 ibevent.device = nesqp->ibqp.device;
3136                                 ibevent.element.qp = &nesqp->ibqp;
3137                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3138                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3139                         }
3140                         break;
3141                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3142                         context <<= 1;
3143                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3144                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3145                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3146                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3147                         if (resource_allocated) {
3148                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3149                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3150                         }
3151                         break;
3152                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3153                         nesqp = nesadapter->qp_table[le32_to_cpu(
3154                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3155                         spin_lock_irqsave(&nesqp->lock, flags);
3156                         nesqp->hw_iwarp_state = iwarp_state;
3157                         nesqp->hw_tcp_state = tcp_state;
3158                         nesqp->last_aeq = async_event_id;
3159                         spin_unlock_irqrestore(&nesqp->lock, flags);
3160                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3161                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3162                                         nesqp->hwqp.qp_id);
3163                         if (nesqp->ibqp.event_handler) {
3164                                 ibevent.device = nesqp->ibqp.device;
3165                                 ibevent.element.qp = &nesqp->ibqp;
3166                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3167                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3168                         }
3169                         /* tell cm to disconnect, cm will queue work to thread */
3170                         nes_cm_disconn(nesqp);
3171                         break;
3172                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3173                         nesqp = *((struct nes_qp **)&context);
3174                         spin_lock_irqsave(&nesqp->lock, flags);
3175                         nesqp->hw_iwarp_state = iwarp_state;
3176                         nesqp->hw_tcp_state = tcp_state;
3177                         nesqp->last_aeq = async_event_id;
3178                         spin_unlock_irqrestore(&nesqp->lock, flags);
3179                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3180                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3181                                         nesqp->hwqp.qp_id);
3182                         if (nesqp->ibqp.event_handler) {
3183                                 ibevent.device = nesqp->ibqp.device;
3184                                 ibevent.element.qp = &nesqp->ibqp;
3185                                 ibevent.event = IB_EVENT_QP_FATAL;
3186                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3187                         }
3188                         /* tell cm to disconnect, cm will queue work to thread */
3189                         nes_cm_disconn(nesqp);
3190                         break;
3191                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3192                         nesqp = *((struct nes_qp **)&context);
3193                         spin_lock_irqsave(&nesqp->lock, flags);
3194                         nesqp->hw_iwarp_state = iwarp_state;
3195                         nesqp->hw_tcp_state = tcp_state;
3196                         nesqp->last_aeq = async_event_id;
3197                         spin_unlock_irqrestore(&nesqp->lock, flags);
3198                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3199                                         " event on QP%u \n  Q2 Data:\n",
3200                                         nesqp->hwqp.qp_id);
3201                         if (nesqp->ibqp.event_handler) {
3202                                 ibevent.device = nesqp->ibqp.device;
3203                                 ibevent.element.qp = &nesqp->ibqp;
3204                                 ibevent.event = IB_EVENT_QP_FATAL;
3205                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3206                         }
3207                         /* tell cm to disconnect, cm will queue work to thread */
3208                         nes_cm_disconn(nesqp);
3209                         break;
3210                         /* TODO: additional AEs need to be here */
3211                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3212                         nesqp = *((struct nes_qp **)&context);
3213                         spin_lock_irqsave(&nesqp->lock, flags);
3214                         nesqp->hw_iwarp_state = iwarp_state;
3215                         nesqp->hw_tcp_state = tcp_state;
3216                         nesqp->last_aeq = async_event_id;
3217                         spin_unlock_irqrestore(&nesqp->lock, flags);
3218                         if (nesqp->ibqp.event_handler) {
3219                                 ibevent.device = nesqp->ibqp.device;
3220                                 ibevent.element.qp = &nesqp->ibqp;
3221                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3222                                 nesqp->ibqp.event_handler(&ibevent,
3223                                                 nesqp->ibqp.qp_context);
3224                         }
3225                         nes_cm_disconn(nesqp);
3226                         break;
3227                 default:
3228                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3229                                         async_event_id);
3230                         break;
3231         }
3232
3233 }
3234
3235
3236 /**
3237  * nes_iwarp_ce_handler
3238  */
3239 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3240 {
3241         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3242
3243         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3244                         nescq->hw_cq.cq_number); */
3245         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3246
3247         if (nescq->ibcq.comp_handler)
3248                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3249
3250         return;
3251 }
3252
3253
3254 /**
3255  * nes_manage_apbvt()
3256  */
3257 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3258                 u32 nic_index, u32 add_port)
3259 {
3260         struct nes_device *nesdev = nesvnic->nesdev;
3261         struct nes_hw_cqp_wqe *cqp_wqe;
3262         struct nes_cqp_request *cqp_request;
3263         int ret = 0;
3264         u16 major_code;
3265
3266         /* Send manage APBVT request to CQP */
3267         cqp_request = nes_get_cqp_request(nesdev);
3268         if (cqp_request == NULL) {
3269                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3270                 return -ENOMEM;
3271         }
3272         cqp_request->waiting = 1;
3273         cqp_wqe = &cqp_request->cqp_wqe;
3274
3275         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3276                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3277                         accel_local_port, accel_local_port, nic_index);
3278
3279         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3280         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3281                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3282         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3283                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3284
3285         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3286
3287         atomic_set(&cqp_request->refcount, 2);
3288         nes_post_cqp_request(nesdev, cqp_request);
3289
3290         if (add_port == NES_MANAGE_APBVT_ADD)
3291                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3292                                 NES_EVENT_TIMEOUT);
3293         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3294                         ret, cqp_request->major_code, cqp_request->minor_code);
3295         major_code = cqp_request->major_code;
3296
3297         nes_put_cqp_request(nesdev, cqp_request);
3298
3299         if (!ret)
3300                 return -ETIME;
3301         else if (major_code)
3302                 return -EIO;
3303         else
3304                 return 0;
3305 }
3306
3307
3308 /**
3309  * nes_manage_arp_cache
3310  */
3311 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3312                 u32 ip_addr, u32 action)
3313 {
3314         struct nes_hw_cqp_wqe *cqp_wqe;
3315         struct nes_vnic *nesvnic = netdev_priv(netdev);
3316         struct nes_device *nesdev;
3317         struct nes_cqp_request *cqp_request;
3318         int arp_index;
3319
3320         nesdev = nesvnic->nesdev;
3321         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3322         if (arp_index == -1) {
3323                 return;
3324         }
3325
3326         /* update the ARP entry */
3327         cqp_request = nes_get_cqp_request(nesdev);
3328         if (cqp_request == NULL) {
3329                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3330                 return;
3331         }
3332         cqp_request->waiting = 0;
3333         cqp_wqe = &cqp_request->cqp_wqe;
3334         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3335
3336         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3337                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3338         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3339                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3340         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3341
3342         if (action == NES_ARP_ADD) {
3343                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3344                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3345                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3346                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3347                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3348                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3349         } else {
3350                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3351                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3352         }
3353
3354         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3355                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3356
3357         atomic_set(&cqp_request->refcount, 1);
3358         nes_post_cqp_request(nesdev, cqp_request);
3359 }
3360
3361
3362 /**
3363  * flush_wqes
3364  */
3365 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3366                 u32 which_wq, u32 wait_completion)
3367 {
3368         struct nes_cqp_request *cqp_request;
3369         struct nes_hw_cqp_wqe *cqp_wqe;
3370         int ret;
3371
3372         cqp_request = nes_get_cqp_request(nesdev);
3373         if (cqp_request == NULL) {
3374                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3375                 return;
3376         }
3377         if (wait_completion) {
3378                 cqp_request->waiting = 1;
3379                 atomic_set(&cqp_request->refcount, 2);
3380         } else {
3381                 cqp_request->waiting = 0;
3382         }
3383         cqp_wqe = &cqp_request->cqp_wqe;
3384         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3385
3386         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3387                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3388         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3389
3390         nes_post_cqp_request(nesdev, cqp_request);
3391
3392         if (wait_completion) {
3393                 /* Wait for CQP */
3394                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3395                                 NES_EVENT_TIMEOUT);
3396                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3397                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3398                                 ret, cqp_request->major_code, cqp_request->minor_code);
3399                 nes_put_cqp_request(nesdev, cqp_request);
3400         }
3401 }