SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static u32 crit_err_count;
  46 u32 int_mod_timer_init;
  47 u32 int_mod_cq_depth_256;
  48 u32 int_mod_cq_depth_128;
  49 u32 int_mod_cq_depth_32;
  50 u32 int_mod_cq_depth_24;
  51 u32 int_mod_cq_depth_16;
  52 u32 int_mod_cq_depth_4;
  53 u32 int_mod_cq_depth_1;
  54
  55 #include "nes_cm.h"
  56
  57 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  58 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  59 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  60                            u8 OneG_Mode);
  61 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  62 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  63 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  64 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  65                                    struct nes_hw_aeqe *aeqe);
  66 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  67 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  68
  69 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  70 static unsigned char *nes_iwarp_state_str[] = {
  71         "Non-Existant",
  72         "Idle",
  73         "RTS",
  74         "Closing",
  75         "RSVD1",
  76         "Terminate",
  77         "Error",
  78         "RSVD2",
  79 };
  80
  81 static unsigned char *nes_tcp_state_str[] = {
  82         "Non-Existant",
  83         "Closed",
  84         "Listen",
  85         "SYN Sent",
  86         "SYN Rcvd",
  87         "Established",
  88         "Close Wait",
  89         "FIN Wait 1",
  90         "Closing",
  91         "Last Ack",
  92         "FIN Wait 2",
  93         "Time Wait",
  94         "RSVD1",
  95         "RSVD2",
  96         "RSVD3",
  97         "RSVD4",
  98 };
  99 #endif
 100
 101
 102 /**
 103  * nes_nic_init_timer_defaults
 104  */
 105 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 106 {
 107         unsigned long flags;
 108         struct nes_adapter *nesadapter = nesdev->nesadapter;
 109         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 110
 111         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 112
 113         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 114         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 115         if (jumbomode) {
 116                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 117                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 118                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 119         } else {
 120                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 121                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 122                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 123         }
 124
 125         /* todo use netdev->mtu to set thresholds */
 126         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 127 }
 128
 129
 130 /**
 131  * nes_nic_init_timer
 132  */
 133 static void  nes_nic_init_timer(struct nes_device *nesdev)
 134 {
 135         unsigned long flags;
 136         struct nes_adapter *nesadapter = nesdev->nesadapter;
 137         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 138
 139         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 140
 141         if (shared_timer->timer_in_use_old == 0) {
 142                 nesdev->deepcq_count = 0;
 143                 shared_timer->timer_direction_upward = 0;
 144                 shared_timer->timer_direction_downward = 0;
 145                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 146                 shared_timer->timer_in_use_old = 0;
 147
 148         }
 149         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 150                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 151                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 152                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 153         }
 154         /* todo use netdev->mtu to set thresholds */
 155         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 156 }
 157
 158
 159 /**
 160  * nes_nic_tune_timer
 161  */
 162 static void nes_nic_tune_timer(struct nes_device *nesdev)
 163 {
 164         unsigned long flags;
 165         struct nes_adapter *nesadapter = nesdev->nesadapter;
 166         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 167         u16 cq_count = nesdev->currcq_count;
 168
 169         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 170
 171         if (shared_timer->cq_count_old <= cq_count)
 172                 shared_timer->cq_direction_downward = 0;
 173         else
 174                 shared_timer->cq_direction_downward++;
 175         shared_timer->cq_count_old = cq_count;
 176         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 177                 if (cq_count <= shared_timer->threshold_low &&
 178                     shared_timer->threshold_low > 4) {
 179                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 180                         shared_timer->cq_direction_downward=0;
 181                         nesdev->currcq_count = 0;
 182                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 183                         return;
 184                 }
 185         }
 186
 187         if (cq_count > 1) {
 188                 nesdev->deepcq_count += cq_count;
 189                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 190                         shared_timer->timer_direction_upward++;
 191                         shared_timer->timer_direction_downward = 0;
 192                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 193                         shared_timer->timer_direction_upward = 0;
 194                         shared_timer->timer_direction_downward = 0;
 195                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 196                         shared_timer->timer_direction_downward++;
 197                         shared_timer->timer_direction_upward = 0;
 198                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 199                         shared_timer->timer_in_use -= 2;
 200                         shared_timer->timer_direction_upward = 0;
 201                         shared_timer->timer_direction_downward++;
 202                 } else {
 203                         shared_timer->timer_in_use -= 4;
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward++;
 206                 }
 207
 208                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 209                         shared_timer->timer_in_use += 3;
 210                         shared_timer->timer_direction_upward = 0;
 211                         shared_timer->timer_direction_downward = 0;
 212                 }
 213                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 214                         shared_timer->timer_in_use -= 4 ;
 215                         shared_timer->timer_direction_downward = 0;
 216                         shared_timer->timer_direction_upward = 0;
 217                 }
 218         }
 219
 220         /* boundary checking */
 221         if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
 222                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
 223         else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
 224                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
 225         }
 226
 227         nesdev->currcq_count = 0;
 228
 229         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 230 }
 231
 232
 233 /**
 234  * nes_init_adapter - initialize adapter
 235  */
 236 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 237         struct nes_adapter *nesadapter = NULL;
 238         unsigned long num_pds;
 239         u32 u32temp;
 240         u32 port_count;
 241         u16 max_rq_wrs;
 242         u16 max_sq_wrs;
 243         u32 max_mr;
 244         u32 max_256pbl;
 245         u32 max_4kpbl;
 246         u32 max_qp;
 247         u32 max_irrq;
 248         u32 max_cq;
 249         u32 hte_index_mask;
 250         u32 adapter_size;
 251         u32 arp_table_size;
 252         u16 vendor_id;
 253         u8  OneG_Mode;
 254         u8  func_index;
 255
 256         /* search the list of existing adapters */
 257         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 258                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 259                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 260                                 nesdev->pcidev->devfn,
 261                                 PCI_SLOT(nesadapter->devfn),
 262                                 nesadapter->bus_number,
 263                                 PCI_SLOT(nesdev->pcidev->devfn),
 264                                 nesdev->pcidev->bus->number );
 265                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 266                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 267                         nesadapter->ref_count++;
 268                         return nesadapter;
 269                 }
 270         }
 271
 272         /* no adapter found */
 273         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 274         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 275                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 276                                 hw_rev);
 277                 return NULL;
 278         }
 279
 280         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 281                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 282                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 283                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 284                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 285
 286         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 287
 288
 289         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 290                 return NULL;
 291         if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
 292                 return NULL;
 293         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 294
 295         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 296         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 297
 298         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 299         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 300                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 301                                 max_qp, u32temp);
 302                 max_qp = (u32)1 << (u32temp & 0x001f);
 303         }
 304
 305         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 306         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 307                         max_qp, hte_index_mask);
 308
 309         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 310
 311         max_irrq = 1 << (u32temp & 0x001f);
 312
 313         if (max_qp > max_irrq) {
 314                 max_qp = max_irrq;
 315                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 316                                 max_qp);
 317         }
 318
 319         /* there should be no reason to allocate more pds than qps */
 320         if (num_pds > max_qp)
 321                 num_pds = max_qp;
 322
 323         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 324         max_mr = (u32)8192 << (u32temp & 0x7);
 325
 326         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 327         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 328         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 329         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 330
 331         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 332         arp_table_size = 1 << u32temp;
 333
 334         adapter_size = (sizeof(struct nes_adapter) +
 335                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 336         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 337         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 340         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 341         adapter_size += sizeof(struct nes_qp **) * max_qp;
 342
 343         /* allocate a new adapter struct */
 344         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 345         if (nesadapter == NULL) {
 346                 return NULL;
 347         }
 348
 349         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 350                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 351
 352         /* populate the new nesadapter */
 353         nesadapter->devfn = nesdev->pcidev->devfn;
 354         nesadapter->bus_number = nesdev->pcidev->bus->number;
 355         nesadapter->ref_count = 1;
 356         nesadapter->timer_int_req = 0xffff0000;
 357         nesadapter->OneG_Mode = OneG_Mode;
 358         nesadapter->doorbell_start = nesdev->doorbell_region;
 359
 360         /* nesadapter->tick_delta = clk_divisor; */
 361         nesadapter->hw_rev = hw_rev;
 362         nesadapter->port_count = port_count;
 363
 364         nesadapter->max_qp = max_qp;
 365         nesadapter->hte_index_mask = hte_index_mask;
 366         nesadapter->max_irrq = max_irrq;
 367         nesadapter->max_mr = max_mr;
 368         nesadapter->max_256pbl = max_256pbl - 1;
 369         nesadapter->max_4kpbl = max_4kpbl - 1;
 370         nesadapter->max_cq = max_cq;
 371         nesadapter->free_256pbl = max_256pbl - 1;
 372         nesadapter->free_4kpbl = max_4kpbl - 1;
 373         nesadapter->max_pd = num_pds;
 374         nesadapter->arp_table_size = arp_table_size;
 375
 376         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 377         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 378                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 379                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 380                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 381         } else {
 382                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 383                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 384                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 385                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 386         }
 387         /* Setup and enable the periodic timer */
 388         if (nesadapter->et_rx_coalesce_usecs_irq)
 389                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 390                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 391         else
 392                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 393
 394         nesadapter->base_pd = 1;
 395
 396         nesadapter->device_cap_flags =
 397                 IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
 398
 399         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 400                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 401         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 402         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 403         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 404         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 405         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 406
 407
 408         /* mark the usual suspect QPs and CQs as in use */
 409         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 410                 set_bit(u32temp, nesadapter->allocated_qps);
 411                 set_bit(u32temp, nesadapter->allocated_cqs);
 412         }
 413
 414         for (u32temp = 0; u32temp < 20; u32temp++)
 415                 set_bit(u32temp, nesadapter->allocated_pds);
 416         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 417
 418         max_rq_wrs = ((u32temp >> 8) & 3);
 419         switch (max_rq_wrs) {
 420                 case 0:
 421                         max_rq_wrs = 4;
 422                         break;
 423                 case 1:
 424                         max_rq_wrs = 16;
 425                         break;
 426                 case 2:
 427                         max_rq_wrs = 32;
 428                         break;
 429                 case 3:
 430                         max_rq_wrs = 512;
 431                         break;
 432         }
 433
 434         max_sq_wrs = (u32temp & 3);
 435         switch (max_sq_wrs) {
 436                 case 0:
 437                         max_sq_wrs = 4;
 438                         break;
 439                 case 1:
 440                         max_sq_wrs = 16;
 441                         break;
 442                 case 2:
 443                         max_sq_wrs = 32;
 444                         break;
 445                 case 3:
 446                         max_sq_wrs = 512;
 447                         break;
 448         }
 449         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 450         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 451
 452         nesadapter->max_sge = 4;
 453         nesadapter->max_cqe = 32767;
 454
 455         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 456                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 457                 kfree(nesadapter);
 458                 return NULL;
 459         }
 460
 461         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 462         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 463                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 464
 465         /* setup port configuration */
 466         if (nesadapter->port_count == 1) {
 467                 u32temp = 0x00000000;
 468                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 469                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 470                 else
 471                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 472         } else {
 473                 if (nesadapter->port_count == 2)
 474                         u32temp = 0x00000044;
 475                 else
 476                         u32temp = 0x000000e4;
 477                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 478         }
 479
 480         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
 481         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 482                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 483
 484         spin_lock_init(&nesadapter->resource_lock);
 485         spin_lock_init(&nesadapter->phy_lock);
 486         spin_lock_init(&nesadapter->pbl_lock);
 487         spin_lock_init(&nesadapter->periodic_timer_lock);
 488
 489         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 490         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 491         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 492         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 493
 494         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 495                 u32 pcs_control_status0, pcs_control_status1;
 496                 u32 reset_value;
 497                 u32 i = 0;
 498                 u32 int_cnt = 0;
 499                 u32 ext_cnt = 0;
 500                 unsigned long flags;
 501                 u32 j = 0;
 502
 503                 pcs_control_status0 = nes_read_indexed(nesdev,
 504                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 505                 pcs_control_status1 = nes_read_indexed(nesdev,
 506                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 507
 508                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 509                         pcs_control_status0 = nes_read_indexed(nesdev,
 510                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 511                         pcs_control_status1 = nes_read_indexed(nesdev,
 512                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 513                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 514                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 515                                 int_cnt++;
 516                         msleep(1);
 517                 }
 518                 if (int_cnt > 1) {
 519                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 520                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 521                         mh_detected++;
 522                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 523                         reset_value |= 0x0000003d;
 524                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 525
 526                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 527                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 528                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 529
 530                         pcs_control_status0 = nes_read_indexed(nesdev,
 531                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 532                         pcs_control_status1 = nes_read_indexed(nesdev,
 533                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 534
 535                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 536                                 pcs_control_status0 = nes_read_indexed(nesdev,
 537                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 538                                 pcs_control_status1 = nes_read_indexed(nesdev,
 539                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 540                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 541                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 542                                         if (++ext_cnt > int_cnt) {
 543                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 544                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 545                                                                 0x0000F0C8);
 546                                                 mh_detected++;
 547                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 548                                                 reset_value |= 0x0000003d;
 549                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 550
 551                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 552                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 553                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 554                                                 break;
 555                                         }
 556                                 }
 557                                 msleep(1);
 558                         }
 559                 }
 560         }
 561
 562         if (nesadapter->hw_rev == NE020_REV) {
 563                 init_timer(&nesadapter->mh_timer);
 564                 nesadapter->mh_timer.function = nes_mh_fix;
 565                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 566                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 567                 add_timer(&nesadapter->mh_timer);
 568         } else {
 569                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 570         }
 571
 572         init_timer(&nesadapter->lc_timer);
 573         nesadapter->lc_timer.function = nes_clc;
 574         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 575         nesadapter->lc_timer.data = (unsigned long)nesdev;
 576         add_timer(&nesadapter->lc_timer);
 577
 578         list_add_tail(&nesadapter->list, &nes_adapter_list);
 579
 580         for (func_index = 0; func_index < 8; func_index++) {
 581                 pci_bus_read_config_word(nesdev->pcidev->bus,
 582                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 583                                         func_index), 0, &vendor_id);
 584                 if (vendor_id == 0xffff)
 585                         break;
 586         }
 587         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 588                 func_index, pci_name(nesdev->pcidev));
 589         nesadapter->adapter_fcn_count = func_index;
 590
 591         return nesadapter;
 592 }
 593
 594
 595 /**
 596  * nes_reset_adapter_ne020
 597  */
 598 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 599 {
 600         u32 port_count;
 601         u32 u32temp;
 602         u32 i;
 603
 604         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 605         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 606         /* TODO: assuming that both SERDES are set the same for now */
 607         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 608         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 609                         u32temp, port_count);
 610         if (*OneG_Mode)
 611                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 612         u32temp &= 0xff00ffc0;
 613         switch (port_count) {
 614                 case 1:
 615                         u32temp |= 0x00ee0000;
 616                         break;
 617                 case 2:
 618                         u32temp |= 0x00cc0000;
 619                         break;
 620                 case 4:
 621                         u32temp |= 0x00000000;
 622                         break;
 623                 default:
 624                         return 0;
 625                         break;
 626         }
 627
 628         /* check and do full reset if needed */
 629         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 630                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 631                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 632
 633                 i = 0;
 634                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 635                         mdelay(1);
 636                 if (i >= 10000) {
 637                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 638                         return 0;
 639                 }
 640
 641                 i = 0;
 642                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 643                         mdelay(1);
 644                 if (i >= 10000) {
 645                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 646                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 647                         return 0;
 648                 }
 649         }
 650
 651         /* port reset */
 652         switch (port_count) {
 653                 case 1:
 654                         u32temp |= 0x00ee0010;
 655                         break;
 656                 case 2:
 657                         u32temp |= 0x00cc0030;
 658                         break;
 659                 case 4:
 660                         u32temp |= 0x00000030;
 661                         break;
 662         }
 663
 664         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 665         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 666
 667         i = 0;
 668         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 669                 mdelay(1);
 670         if (i >= 10000) {
 671                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 672                 return 0;
 673         }
 674
 675         /* serdes 0 */
 676         i = 0;
 677         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 678                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 679                 mdelay(1);
 680         if (i >= 5000) {
 681                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 682                 return 0;
 683         }
 684
 685         /* serdes 1 */
 686         if (port_count > 1) {
 687                 i = 0;
 688                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 689                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 690                         mdelay(1);
 691                 if (i >= 5000) {
 692                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 693                         return 0;
 694                 }
 695         }
 696
 697         return port_count;
 698 }
 699
 700
 701 /**
 702  * nes_init_serdes
 703  */
 704 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 705                            u8 OneG_Mode)
 706 {
 707         int i;
 708         u32 u32temp;
 709
 710         if (hw_rev != NE020_REV) {
 711                 /* init serdes 0 */
 712
 713                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 714                 if (!OneG_Mode)
 715                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 716                 if (port_count > 1) {
 717                         /* init serdes 1 */
 718                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 719                         if (!OneG_Mode)
 720                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 721                         }
 722         } else {
 723                 /* init serdes 0 */
 724                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 725                 i = 0;
 726                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 727                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 728                         mdelay(1);
 729                 if (i >= 5000) {
 730                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 731                         return 1;
 732                 }
 733                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 734                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 735                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 736                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 737                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 738                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 739                 if (OneG_Mode)
 740                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 741                 else
 742                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 743
 744                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 745                 if (port_count > 1) {
 746                         /* init serdes 1 */
 747                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 748                         i = 0;
 749                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 750                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 751                                 mdelay(1);
 752                         if (i >= 5000) {
 753                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 754                                 /* return 1; */
 755                         }
 756                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 757                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 762                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 764                 }
 765         }
 766         return 0;
 767 }
 768
 769
 770 /**
 771  * nes_init_csr_ne020
 772  * Initialize registers for ne020 hardware
 773  */
 774 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 775 {
 776         u32 u32temp;
 777
 778         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 779
 780         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 781         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 782         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 783         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 784         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 785         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 786         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 787         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 788
 789         /* TODO: move these MAC register settings to NIC bringup */
 790         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 791         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 792         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 793         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 794         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 795         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 796         if (port_count > 1) {
 797                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 798                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 799                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 800                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 801                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 802                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 803                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 804         }
 805         if (port_count > 2) {
 806                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 807                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 808                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 809                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 810                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 811                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 812                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 813
 814                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 815                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 816                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 817                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 818                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 819                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 820                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 821         }
 822
 823         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 824         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 825         nes_write_indexed(nesdev, 0x00005004, 0x00020001);
 826         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 827         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 828         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 829         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 830         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 831
 832         /* TODO: move this to code, get from EEPROM */
 833         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 834         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 835         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 836                                                                                                                 //
 837         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 838         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 839
 840         if (hw_rev != NE020_REV) {
 841                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 842                 u32temp |= 0x80000000;
 843                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 844                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 845                 u32temp &= 0x7fffffff;
 846                 u32temp |= 0x7fff0010;
 847                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 848         }
 849 }
 850
 851
 852 /**
 853  * nes_destroy_adapter - destroy the adapter structure
 854  */
 855 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 856 {
 857         struct nes_adapter *tmp_adapter;
 858
 859         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 860                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 861                                 tmp_adapter);
 862         }
 863
 864         nesadapter->ref_count--;
 865         if (!nesadapter->ref_count) {
 866                 if (nesadapter->hw_rev == NE020_REV) {
 867                         del_timer(&nesadapter->mh_timer);
 868                 }
 869                 del_timer(&nesadapter->lc_timer);
 870
 871                 list_del(&nesadapter->list);
 872                 kfree(nesadapter);
 873         }
 874 }
 875
 876
 877 /**
 878  * nes_init_cqp
 879  */
 880 int nes_init_cqp(struct nes_device *nesdev)
 881 {
 882         struct nes_adapter *nesadapter = nesdev->nesadapter;
 883         struct nes_hw_cqp_qp_context *cqp_qp_context;
 884         struct nes_hw_cqp_wqe *cqp_wqe;
 885         struct nes_hw_ceq *ceq;
 886         struct nes_hw_ceq *nic_ceq;
 887         struct nes_hw_aeq *aeq;
 888         void *vmem;
 889         dma_addr_t pmem;
 890         u32 count=0;
 891         u32 cqp_head;
 892         u64 u64temp;
 893         u32 u32temp;
 894
 895         /* allocate CQP memory */
 896         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 897         /* SQ is 512 byte aligned, others are 256 byte aligned */
 898         nesdev->cqp_mem_size = 512 +
 899                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 900                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 901                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 902                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 903                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 904                         sizeof(struct nes_hw_cqp_qp_context);
 905
 906         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 907                         &nesdev->cqp_pbase);
 908         if (!nesdev->cqp_vbase) {
 909                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 910                 return -ENOMEM;
 911         }
 912         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 913
 914         /* Allocate a twice the number of CQP requests as the SQ size */
 915         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 916                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 917         if (nesdev->nes_cqp_requests == NULL) {
 918                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 919                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 920                                 nesdev->cqp.sq_pbase);
 921                 return -ENOMEM;
 922         }
 923
 924         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 925                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 926
 927         spin_lock_init(&nesdev->cqp.lock);
 928         init_waitqueue_head(&nesdev->cqp.waitq);
 929
 930         /* Setup Various Structures */
 931         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 932                         ~(unsigned long)(512 - 1));
 933         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 934                         ~(unsigned long long)(512 - 1));
 935
 936         nesdev->cqp.sq_vbase = vmem;
 937         nesdev->cqp.sq_pbase = pmem;
 938         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 939         nesdev->cqp.sq_head = 0;
 940         nesdev->cqp.sq_tail = 0;
 941         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 942
 943         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 944         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 945
 946         nesdev->ccq.cq_vbase = vmem;
 947         nesdev->ccq.cq_pbase = pmem;
 948         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 949         nesdev->ccq.cq_head = 0;
 950         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 951         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 952
 953         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 954         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 955
 956         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
 957         ceq = &nesadapter->ceq[nesdev->ceq_index];
 958         ceq->ceq_vbase = vmem;
 959         ceq->ceq_pbase = pmem;
 960         ceq->ceq_size = NES_CCEQ_SIZE;
 961         ceq->ceq_head = 0;
 962
 963         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 964         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 965
 966         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
 967         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
 968         nic_ceq->ceq_vbase = vmem;
 969         nic_ceq->ceq_pbase = pmem;
 970         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
 971         nic_ceq->ceq_head = 0;
 972
 973         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 974         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 975
 976         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
 977         aeq->aeq_vbase = vmem;
 978         aeq->aeq_pbase = pmem;
 979         aeq->aeq_size = nesadapter->max_qp;
 980         aeq->aeq_head = 0;
 981
 982         /* Setup QP Context */
 983         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 984         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 985
 986         cqp_qp_context = vmem;
 987         cqp_qp_context->context_words[0] =
 988                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
 989         cqp_qp_context->context_words[1] = 0;
 990         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
 991         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
 992
 993
 994         /* Write the address to Create CQP */
 995         if ((sizeof(dma_addr_t) > 4)) {
 996                 nes_write_indexed(nesdev,
 997                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 998                                 ((u64)pmem) >> 32);
 999         } else {
1000                 nes_write_indexed(nesdev,
1001                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1002         }
1003         nes_write_indexed(nesdev,
1004                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1005                         (u32)pmem);
1006
1007         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1008         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1009
1010         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1011                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1012                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1013         }
1014
1015         /* Write Create CCQ WQE */
1016         cqp_head = nesdev->cqp.sq_head++;
1017         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1018         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1019         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1020                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1021                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1022         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1023                             (nesdev->ccq.cq_number |
1024                              ((u32)nesdev->ceq_index << 16)));
1025         u64temp = (u64)nesdev->ccq.cq_pbase;
1026         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1027         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1028         u64temp = (unsigned long)&nesdev->ccq;
1029         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1030                         cpu_to_le32((u32)(u64temp >> 1));
1031         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1032                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1033         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1034
1035         /* Write Create CEQ WQE */
1036         cqp_head = nesdev->cqp.sq_head++;
1037         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1038         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1039         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1040                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1041         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1042         u64temp = (u64)ceq->ceq_pbase;
1043         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1044
1045         /* Write Create AEQ WQE */
1046         cqp_head = nesdev->cqp.sq_head++;
1047         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1048         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1049         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1050                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1051         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1052         u64temp = (u64)aeq->aeq_pbase;
1053         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1054
1055         /* Write Create NIC CEQ WQE */
1056         cqp_head = nesdev->cqp.sq_head++;
1057         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1058         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1059         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1060                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1061         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1062         u64temp = (u64)nic_ceq->ceq_pbase;
1063         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1064
1065         /* Poll until CCQP done */
1066         count = 0;
1067         do {
1068                 if (count++ > 1000) {
1069                         printk(KERN_ERR PFX "Error creating CQP\n");
1070                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1071                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1072                         return -1;
1073                 }
1074                 udelay(10);
1075         } while (!(nes_read_indexed(nesdev,
1076                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1077
1078         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1079                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1080
1081         u32temp = 0x04800000;
1082         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1083
1084         /* wait for the CCQ, CEQ, and AEQ to get created */
1085         count = 0;
1086         do {
1087                 if (count++ > 1000) {
1088                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1089                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1090                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1091                         return -1;
1092                 }
1093                 udelay(10);
1094         } while (((nes_read_indexed(nesdev,
1095                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1096
1097         /* dump the QP status value */
1098         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1099                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1100
1101         nesdev->cqp.sq_tail++;
1102
1103         return 0;
1104 }
1105
1106
1107 /**
1108  * nes_destroy_cqp
1109  */
1110 int nes_destroy_cqp(struct nes_device *nesdev)
1111 {
1112         struct nes_hw_cqp_wqe *cqp_wqe;
1113         u32 count = 0;
1114         u32 cqp_head;
1115         unsigned long flags;
1116
1117         do {
1118                 if (count++ > 1000)
1119                         break;
1120                 udelay(10);
1121         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1122
1123         /* Reset CCQ */
1124         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1125                         nesdev->ccq.cq_number);
1126
1127         /* Disable device interrupts */
1128         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1129
1130         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1131
1132         /* Destroy the AEQ */
1133         cqp_head = nesdev->cqp.sq_head++;
1134         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1135         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1136         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1137                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1138         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1139
1140         /* Destroy the NIC CEQ */
1141         cqp_head = nesdev->cqp.sq_head++;
1142         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1143         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1144         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1145                         ((u32)nesdev->nic_ceq_index << 8));
1146
1147         /* Destroy the CEQ */
1148         cqp_head = nesdev->cqp.sq_head++;
1149         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1150         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1151         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1152                         (nesdev->ceq_index << 8));
1153
1154         /* Destroy the CCQ */
1155         cqp_head = nesdev->cqp.sq_head++;
1156         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1157         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1158         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1159         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1160                         ((u32)nesdev->ceq_index << 16));
1161
1162         /* Destroy CQP */
1163         cqp_head = nesdev->cqp.sq_head++;
1164         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1165         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1166         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1167                         NES_CQP_QP_TYPE_CQP);
1168         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1169
1170         barrier();
1171         /* Ring doorbell (5 WQEs) */
1172         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1173
1174         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1175
1176         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1177         count = 0;
1178         do {
1179                 if (count++ > 1000) {
1180                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1181                                         PCI_FUNC(nesdev->pcidev->devfn));
1182                         break;
1183                 }
1184                 udelay(10);
1185         } while (((nes_read_indexed(nesdev,
1186                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1187
1188         /* dump the QP status value */
1189         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1190                         PCI_FUNC(nesdev->pcidev->devfn),
1191                         nes_read_indexed(nesdev,
1192                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1193
1194         kfree(nesdev->nes_cqp_requests);
1195
1196         /* Free the control structures */
1197         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1198                         nesdev->cqp.sq_pbase);
1199
1200         return 0;
1201 }
1202
1203
1204 /**
1205  * nes_init_phy
1206  */
1207 int nes_init_phy(struct nes_device *nesdev)
1208 {
1209         struct nes_adapter *nesadapter = nesdev->nesadapter;
1210         u32 counter = 0;
1211         u32 mac_index = nesdev->mac_index;
1212         u32 tx_config;
1213         u16 phy_data;
1214
1215         if (nesadapter->OneG_Mode) {
1216                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1217                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1218                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1219                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1220                         tx_config |= 0x04;
1221                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1222                 }
1223
1224                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1225                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1226                                 nesadapter->phy_index[mac_index], phy_data);
1227                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index],  0xb000);
1228
1229                 /* Reset the PHY */
1230                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1231                 udelay(100);
1232                 counter = 0;
1233                 do {
1234                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1235                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1236                         if (counter++ > 100) break;
1237                 } while (phy_data & 0x8000);
1238
1239                 /* Setting no phy loopback */
1240                 phy_data &= 0xbfff;
1241                 phy_data |= 0x1140;
1242                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1243                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1244                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1245
1246                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1247                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1248
1249                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1250                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1251
1252                 /* Setting the interrupt mask */
1253                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1254                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1255                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1256
1257                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1258                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1259
1260                 /* turning on flow control */
1261                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1262                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1263                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1264                                 (phy_data & ~(0x03E0)) | 0xc00);
1265                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1266                                 phy_data | 0xc00); */
1267                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1268                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1269
1270                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1271                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1272                 /* Clear Half duplex */
1273                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1274                                 phy_data & ~(0x0100));
1275                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1276                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1277
1278                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1279                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1280         } else {
1281                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
1282                         /* setup 10G MDIO operation */
1283                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1284                         tx_config |= 0x14;
1285                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1286                 }
1287         }
1288         return 0;
1289 }
1290
1291
1292 /**
1293  * nes_replenish_nic_rq
1294  */
1295 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1296 {
1297         unsigned long flags;
1298         dma_addr_t bus_address;
1299         struct sk_buff *skb;
1300         struct nes_hw_nic_rq_wqe *nic_rqe;
1301         struct nes_hw_nic *nesnic;
1302         struct nes_device *nesdev;
1303         u32 rx_wqes_posted = 0;
1304
1305         nesnic = &nesvnic->nic;
1306         nesdev = nesvnic->nesdev;
1307         spin_lock_irqsave(&nesnic->rq_lock, flags);
1308         if (nesnic->replenishing_rq !=0) {
1309                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1310                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1311                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1312                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1313                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1314                         add_timer(&nesvnic->rq_wqes_timer);
1315                 } else
1316                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1317                 return;
1318         }
1319         nesnic->replenishing_rq = 1;
1320         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1321         do {
1322                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1323                 if (skb) {
1324                         skb->dev = nesvnic->netdev;
1325
1326                         bus_address = pci_map_single(nesdev->pcidev,
1327                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1328
1329                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1330                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1331                                         cpu_to_le32(nesvnic->max_frame_size);
1332                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1333                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1334                                         cpu_to_le32((u32)bus_address);
1335                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1336                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1337                         nesnic->rx_skb[nesnic->rq_head] = skb;
1338                         nesnic->rq_head++;
1339                         nesnic->rq_head &= nesnic->rq_size - 1;
1340                         atomic_dec(&nesvnic->rx_skbs_needed);
1341                         barrier();
1342                         if (++rx_wqes_posted == 255) {
1343                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1344                                 rx_wqes_posted = 0;
1345                         }
1346                 } else {
1347                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1348                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1349                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1350                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1351                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1352                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1353                                 add_timer(&nesvnic->rq_wqes_timer);
1354                         } else
1355                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1356                         break;
1357                 }
1358         } while (atomic_read(&nesvnic->rx_skbs_needed));
1359         barrier();
1360         if (rx_wqes_posted)
1361                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1362         nesnic->replenishing_rq = 0;
1363 }
1364
1365
1366 /**
1367  * nes_rq_wqes_timeout
1368  */
1369 static void nes_rq_wqes_timeout(unsigned long parm)
1370 {
1371         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1372         printk("%s: Timer fired.\n", __func__);
1373         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1374         if (atomic_read(&nesvnic->rx_skbs_needed))
1375                 nes_replenish_nic_rq(nesvnic);
1376 }
1377
1378
1379 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1380                                void **tcph, u64 *hdr_flags, void *priv)
1381 {
1382         unsigned int ip_len;
1383         struct iphdr *iph;
1384         skb_reset_network_header(skb);
1385         iph = ip_hdr(skb);
1386         if (iph->protocol != IPPROTO_TCP)
1387                 return -1;
1388         ip_len = ip_hdrlen(skb);
1389         skb_set_transport_header(skb, ip_len);
1390         *tcph = tcp_hdr(skb);
1391
1392         *hdr_flags = LRO_IPV4 | LRO_TCP;
1393         *iphdr = iph;
1394         return 0;
1395 }
1396
1397
1398 /**
1399  * nes_init_nic_qp
1400  */
1401 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1402 {
1403         struct nes_hw_cqp_wqe *cqp_wqe;
1404         struct nes_hw_nic_sq_wqe *nic_sqe;
1405         struct nes_hw_nic_qp_context *nic_context;
1406         struct sk_buff *skb;
1407         struct nes_hw_nic_rq_wqe *nic_rqe;
1408         struct nes_vnic *nesvnic = netdev_priv(netdev);
1409         unsigned long flags;
1410         void *vmem;
1411         dma_addr_t pmem;
1412         u64 u64temp;
1413         int ret;
1414         u32 cqp_head;
1415         u32 counter;
1416         u32 wqe_count;
1417         u8 jumbomode=0;
1418
1419         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1420         nesvnic->nic_mem_size = 256 +
1421                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1422                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1423                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1424                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1425                         sizeof(struct nes_hw_nic_qp_context);
1426
1427         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1428                         &nesvnic->nic_pbase);
1429         if (!nesvnic->nic_vbase) {
1430                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1431                 return -ENOMEM;
1432         }
1433         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1434         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1435                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1436
1437         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1438                         ~(unsigned long)(256 - 1));
1439         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1440                         ~(unsigned long long)(256 - 1));
1441
1442         /* Setup the first Fragment buffers */
1443         nesvnic->nic.first_frag_vbase = vmem;
1444
1445         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1446                 nesvnic->nic.frag_paddr[counter] = pmem;
1447                 pmem += sizeof(struct nes_first_frag);
1448         }
1449
1450         /* setup the SQ */
1451         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1452
1453         nesvnic->nic.sq_vbase = (void *)vmem;
1454         nesvnic->nic.sq_pbase = pmem;
1455         nesvnic->nic.sq_head = 0;
1456         nesvnic->nic.sq_tail = 0;
1457         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1458         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1459                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1460                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1461                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1462                                 NES_NIC_SQ_WQE_COMPLETION);
1463                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1464                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1465                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1466                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1467                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1468                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1469         }
1470
1471         nesvnic->get_cqp_request = nes_get_cqp_request;
1472         nesvnic->post_cqp_request = nes_post_cqp_request;
1473         nesvnic->mcrq_mcast_filter = NULL;
1474
1475         spin_lock_init(&nesvnic->nic.sq_lock);
1476         spin_lock_init(&nesvnic->nic.rq_lock);
1477
1478         /* setup the RQ */
1479         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1480         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1481
1482
1483         nesvnic->nic.rq_vbase = vmem;
1484         nesvnic->nic.rq_pbase = pmem;
1485         nesvnic->nic.rq_head = 0;
1486         nesvnic->nic.rq_tail = 0;
1487         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1488
1489         /* setup the CQ */
1490         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1491         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1492
1493         if (nesdev->nesadapter->netdev_count > 2)
1494                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1495         else
1496                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1497
1498         nesvnic->nic_cq.cq_vbase = vmem;
1499         nesvnic->nic_cq.cq_pbase = pmem;
1500         nesvnic->nic_cq.cq_head = 0;
1501         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1502
1503         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1504
1505         /* Send CreateCQ request to CQP */
1506         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1507         cqp_head = nesdev->cqp.sq_head;
1508
1509         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1510         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1511
1512         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1513                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1514                         ((u32)nesvnic->nic_cq.cq_size << 16));
1515         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1516                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1517         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1518         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1519         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1520         u64temp = (unsigned long)&nesvnic->nic_cq;
1521         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1522         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1523                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1524         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1525         if (++cqp_head >= nesdev->cqp.sq_size)
1526                 cqp_head = 0;
1527         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1528         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1529
1530         /* Send CreateQP request to CQP */
1531         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1532         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1533                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1534                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1535         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1536                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1537                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1538         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1539                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1540         }
1541
1542         u64temp = (u64)nesvnic->nic.sq_pbase;
1543         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1544         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1545         u64temp = (u64)nesvnic->nic.rq_pbase;
1546         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1547         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1548
1549         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1550                         NES_CQP_QP_TYPE_NIC);
1551         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1552         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1553                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1554         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1555
1556         if (++cqp_head >= nesdev->cqp.sq_size)
1557                 cqp_head = 0;
1558         nesdev->cqp.sq_head = cqp_head;
1559
1560         barrier();
1561
1562         /* Ring doorbell (2 WQEs) */
1563         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1564
1565         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1566         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1567                         nesvnic->nic.qp_id);
1568
1569         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1570                         NES_EVENT_TIMEOUT);
1571         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1572                         nesvnic->nic.qp_id, ret);
1573         if (!ret) {
1574                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1575                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1576                                 nesvnic->nic_pbase);
1577                 return -EIO;
1578         }
1579
1580         /* Populate the RQ */
1581         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1582                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1583                 if (!skb) {
1584                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1585
1586                         nes_destroy_nic_qp(nesvnic);
1587                         return -ENOMEM;
1588                 }
1589
1590                 skb->dev = netdev;
1591
1592                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1593                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1594
1595                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1596                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1597                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1598                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1599                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1600                 nesvnic->nic.rx_skb[counter] = skb;
1601         }
1602
1603         wqe_count = NES_NIC_WQ_SIZE - 1;
1604         nesvnic->nic.rq_head = wqe_count;
1605         barrier();
1606         do {
1607                 counter = min(wqe_count, ((u32)255));
1608                 wqe_count -= counter;
1609                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1610         } while (wqe_count);
1611         init_timer(&nesvnic->rq_wqes_timer);
1612         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1613         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1614         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1615         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1616         {
1617                 nes_nic_init_timer(nesdev);
1618                 if (netdev->mtu > 1500)
1619                         jumbomode = 1;
1620                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1621         }
1622         nesvnic->lro_mgr.max_aggr = NES_LRO_MAX_AGGR;
1623         nesvnic->lro_mgr.max_desc = NES_MAX_LRO_DESCRIPTORS;
1624         nesvnic->lro_mgr.lro_arr = nesvnic->lro_desc;
1625         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1626         nesvnic->lro_mgr.features = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1627         nesvnic->lro_mgr.dev = netdev;
1628         nesvnic->lro_mgr.ip_summed = CHECKSUM_UNNECESSARY;
1629         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1630         return 0;
1631 }
1632
1633
1634 /**
1635  * nes_destroy_nic_qp
1636  */
1637 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1638 {
1639         struct nes_device *nesdev = nesvnic->nesdev;
1640         struct nes_hw_cqp_wqe *cqp_wqe;
1641         struct nes_hw_nic_rq_wqe *nic_rqe;
1642         u64 wqe_frag;
1643         u32 cqp_head;
1644         unsigned long flags;
1645         int ret;
1646
1647         /* Free remaining NIC receive buffers */
1648         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1649                 nic_rqe = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1650                 wqe_frag = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1651                 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1652                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1653                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1654                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1655                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1656         }
1657
1658         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1659
1660         /* Destroy NIC QP */
1661         cqp_head = nesdev->cqp.sq_head;
1662         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1663         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1664
1665         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1666                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1667         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1668                 nesvnic->nic.qp_id);
1669
1670         if (++cqp_head >= nesdev->cqp.sq_size)
1671                 cqp_head = 0;
1672
1673         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1674
1675         /* Destroy NIC CQ */
1676         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1677         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1678                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1679         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1680                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1681
1682         if (++cqp_head >= nesdev->cqp.sq_size)
1683                 cqp_head = 0;
1684
1685         nesdev->cqp.sq_head = cqp_head;
1686         barrier();
1687
1688         /* Ring doorbell (2 WQEs) */
1689         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1690
1691         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1692         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1693                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1694                         cqp_head, nesdev->cqp.sq_head,
1695                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1696
1697         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1698                         NES_EVENT_TIMEOUT);
1699
1700         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1701                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1702                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1703         if (!ret) {
1704                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1705                                 nesvnic->nic.qp_id);
1706         }
1707
1708         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1709                         nesvnic->nic_pbase);
1710 }
1711
1712 /**
1713  * nes_napi_isr
1714  */
1715 int nes_napi_isr(struct nes_device *nesdev)
1716 {
1717         struct nes_adapter *nesadapter = nesdev->nesadapter;
1718         u32 int_stat;
1719
1720         if (nesdev->napi_isr_ran) {
1721                 /* interrupt status has already been read in ISR */
1722                 int_stat = nesdev->int_stat;
1723         } else {
1724                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1725                 nesdev->int_stat = int_stat;
1726                 nesdev->napi_isr_ran = 1;
1727         }
1728
1729         int_stat &= nesdev->int_req;
1730         /* iff NIC, process here, else wait for DPC */
1731         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1732                 nesdev->napi_isr_ran = 0;
1733                 nes_write32(nesdev->regs+NES_INT_STAT,
1734                                 (int_stat &
1735                                 ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1736
1737                 /* Process the CEQs */
1738                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1739
1740                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1741                                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1742                                           ((nesadapter->et_use_adaptive_rx_coalesce) &&
1743                                            (nesdev->deepcq_count > nesadapter->et_pkt_rate_low)))) ) {
1744                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1745                                 /* Enable Periodic timer interrupts */
1746                                 nesdev->int_req |= NES_INT_TIMER;
1747                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1748                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1749                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1750                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1751                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1752                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1753                         }
1754
1755                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1756                         {
1757                                 nes_nic_init_timer(nesdev);
1758                         }
1759                         /* Enable interrupts, except CEQs */
1760                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1761                 } else {
1762                         /* Enable interrupts, make sure timer is off */
1763                         nesdev->int_req &= ~NES_INT_TIMER;
1764                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1765                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1766                 }
1767                 nesdev->deepcq_count = 0;
1768                 return 1;
1769         } else {
1770                 return 0;
1771         }
1772 }
1773
1774
1775 /**
1776  * nes_dpc
1777  */
1778 void nes_dpc(unsigned long param)
1779 {
1780         struct nes_device *nesdev = (struct nes_device *)param;
1781         struct nes_adapter *nesadapter = nesdev->nesadapter;
1782         u32 counter;
1783         u32 loop_counter = 0;
1784         u32 int_status_bit;
1785         u32 int_stat;
1786         u32 timer_stat;
1787         u32 temp_int_stat;
1788         u32 intf_int_stat;
1789         u32 debug_error;
1790         u32 processed_intf_int = 0;
1791         u16 processed_timer_int = 0;
1792         u16 completion_ints = 0;
1793         u16 timer_ints = 0;
1794
1795         /* nes_debug(NES_DBG_ISR, "\n"); */
1796
1797         do {
1798                 timer_stat = 0;
1799                 if (nesdev->napi_isr_ran) {
1800                         nesdev->napi_isr_ran = 0;
1801                         int_stat = nesdev->int_stat;
1802                 } else
1803                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1804                 if (processed_intf_int != 0)
1805                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
1806                 else
1807                         int_stat &= nesdev->int_req;
1808                 if (processed_timer_int == 0) {
1809                         processed_timer_int = 1;
1810                         if (int_stat & NES_INT_TIMER) {
1811                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1812                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
1813                                         int_stat &= ~NES_INT_TIMER;
1814                                 }
1815                         }
1816                 } else {
1817                         int_stat &= ~NES_INT_TIMER;
1818                 }
1819
1820                 if (int_stat) {
1821                         if (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1822                                         NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)) {
1823                                 /* Ack the interrupts */
1824                                 nes_write32(nesdev->regs+NES_INT_STAT,
1825                                                 (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1826                                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1827                         }
1828
1829                         temp_int_stat = int_stat;
1830                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1831                                 if (int_stat & int_status_bit) {
1832                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1833                                         temp_int_stat &= ~int_status_bit;
1834                                         completion_ints = 1;
1835                                 }
1836                                 if (!(temp_int_stat & 0x0000ffff))
1837                                         break;
1838                                 int_status_bit <<= 1;
1839                         }
1840
1841                         /* Process the AEQ for this pci function */
1842                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1843                         if (int_stat & int_status_bit) {
1844                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1845                         }
1846
1847                         /* Process the MAC interrupt for this pci function */
1848                         int_status_bit = 1 << (24 + nesdev->mac_index);
1849                         if (int_stat & int_status_bit) {
1850                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
1851                         }
1852
1853                         if (int_stat & NES_INT_TIMER) {
1854                                 if (timer_stat & nesdev->timer_int_req) {
1855                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
1856                                                         (timer_stat & nesdev->timer_int_req) |
1857                                                         ~(nesdev->nesadapter->timer_int_req));
1858                                         timer_ints = 1;
1859                                 }
1860                         }
1861
1862                         if (int_stat & NES_INT_INTF) {
1863                                 processed_intf_int = 1;
1864                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1865                                 intf_int_stat &= nesdev->intf_int_req;
1866                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1867                                         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1868                                         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1869                                                         (u16)debug_error);
1870                                         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1871                                                         0x01010000 | (debug_error & 0x0000ffff));
1872                                         /* BUG(); */
1873                                         if (crit_err_count++ > 10)
1874                                                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1875                                 }
1876                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
1877                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
1878                                         BUG();
1879                                 }
1880                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
1881                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
1882                                         BUG();
1883                                 }
1884                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
1885                         }
1886
1887                         if (int_stat & NES_INT_TSW) {
1888                         }
1889                 }
1890                 /* Don't use the interface interrupt bit stay in loop */
1891                 int_stat &= ~NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1892                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3;
1893         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
1894
1895         if (timer_ints == 1) {
1896                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
1897                         if (completion_ints == 0) {
1898                                 nesdev->timer_only_int_count++;
1899                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
1900                                         nesdev->timer_only_int_count = 0;
1901                                         nesdev->int_req &= ~NES_INT_TIMER;
1902                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1903                                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1904                                 } else {
1905                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1906                                 }
1907                         } else {
1908                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1909                                 {
1910                                         nes_nic_init_timer(nesdev);
1911                                 }
1912                                 nesdev->timer_only_int_count = 0;
1913                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1914                         }
1915                 } else {
1916                         nesdev->timer_only_int_count = 0;
1917                         nesdev->int_req &= ~NES_INT_TIMER;
1918                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1919                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1920                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1921                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1922                 }
1923         } else {
1924                 if ( (completion_ints == 1) &&
1925                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
1926                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1927                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
1928                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
1929                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
1930                         nesdev->timer_only_int_count = 0;
1931                         nesdev->int_req |= NES_INT_TIMER;
1932                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1933                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1934                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1935                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1936                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1937                 } else {
1938                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1939                 }
1940         }
1941         nesdev->deepcq_count = 0;
1942 }
1943
1944
1945 /**
1946  * nes_process_ceq
1947  */
1948 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
1949 {
1950         u64 u64temp;
1951         struct nes_hw_cq *cq;
1952         u32 head;
1953         u32 ceq_size;
1954
1955         /* nes_debug(NES_DBG_CQ, "\n"); */
1956         head = ceq->ceq_head;
1957         ceq_size = ceq->ceq_size;
1958
1959         do {
1960                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
1961                                 NES_CEQE_VALID) {
1962                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX])))<<32) |
1963                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
1964                         u64temp <<= 1;
1965                         cq = *((struct nes_hw_cq **)&u64temp);
1966                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
1967                         barrier();
1968                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
1969
1970                         /* call the event handler */
1971                         cq->ce_handler(nesdev, cq);
1972
1973                         if (++head >= ceq_size)
1974                                 head = 0;
1975                 } else {
1976                         break;
1977                 }
1978
1979         } while (1);
1980
1981         ceq->ceq_head = head;
1982 }
1983
1984
1985 /**
1986  * nes_process_aeq
1987  */
1988 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
1989 {
1990 //      u64 u64temp;
1991         u32 head;
1992         u32 aeq_size;
1993         u32 aeqe_misc;
1994         u32 aeqe_cq_id;
1995         struct nes_hw_aeqe volatile *aeqe;
1996
1997         head = aeq->aeq_head;
1998         aeq_size = aeq->aeq_size;
1999
2000         do {
2001                 aeqe = &aeq->aeq_vbase[head];
2002                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2003                         break;
2004                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2005                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2006                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2007                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2008                                 /* dealing with an accelerated QP related AE */
2009 //                              u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])))<<32) |
2010 //                                      ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2011                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2012                         } else {
2013                                 /* TODO: dealing with a CQP related AE */
2014                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2015                                                 (u16)(aeqe_misc >> 16));
2016                         }
2017                 }
2018
2019                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2020
2021                 if (++head >= aeq_size)
2022                         head = 0;
2023         }
2024         while (1);
2025         aeq->aeq_head = head;
2026 }
2027
2028 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2029 {
2030         struct nes_adapter *nesadapter = nesdev->nesadapter;
2031         u32 reset_value;
2032         u32 i=0;
2033         u32 u32temp;
2034
2035         if (nesadapter->hw_rev == NE020_REV) {
2036                 return;
2037         }
2038         mh_detected++;
2039
2040         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2041
2042         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2043                 reset_value |= 0x0000001d;
2044         else
2045                 reset_value |= 0x0000002d;
2046
2047         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2048                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2049                         nesadapter->link_interrupt_count[0] = 0;
2050                         nesadapter->link_interrupt_count[1] = 0;
2051                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2052                         if (0x00000040 & u32temp)
2053                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2054                         else
2055                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2056
2057                         reset_value |= 0x0000003d;
2058                 }
2059                 nesadapter->link_interrupt_count[mac_index] = 0;
2060         }
2061
2062         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2063
2064         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2065                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2066
2067         if (0x0000003d == (reset_value & 0x0000003d)) {
2068                 u32 pcs_control_status0, pcs_control_status1;
2069
2070                 for (i = 0; i < 10; i++) {
2071                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2072                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2073                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2074                              && (pcs_control_status0 & 0x00100000))
2075                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2076                                 && (pcs_control_status1 & 0x00100000)))
2077                                 continue;
2078                         else
2079                                 break;
2080                 }
2081                 if (10 == i) {
2082                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2083                         if (0x00000040 & u32temp)
2084                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2085                         else
2086                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2087
2088                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2089
2090                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2091                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2092                 }
2093         }
2094 }
2095
2096 /**
2097  * nes_process_mac_intr
2098  */
2099 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2100 {
2101         unsigned long flags;
2102         u32 pcs_control_status;
2103         struct nes_adapter *nesadapter = nesdev->nesadapter;
2104         struct nes_vnic *nesvnic;
2105         u32 mac_status;
2106         u32 mac_index = nesdev->mac_index;
2107         u32 u32temp;
2108         u16 phy_data;
2109         u16 temp_phy_data;
2110
2111         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2112         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2113                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2114                 return;
2115         }
2116         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2117         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2118
2119         /* ack the MAC interrupt */
2120         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2121         /* Clear the interrupt */
2122         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2123
2124         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2125
2126         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2127                 nesdev->link_status_interrupts++;
2128                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2129                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2130                         nes_reset_link(nesdev, mac_index);
2131                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2132                 }
2133                 /* read the PHY interrupt status register */
2134                 if (nesadapter->OneG_Mode) {
2135                         do {
2136                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2137                                                 nesadapter->phy_index[mac_index], &phy_data);
2138                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2139                                                 nesadapter->phy_index[mac_index], phy_data);
2140                         } while (phy_data&0x8000);
2141
2142                         temp_phy_data = 0;
2143                         do {
2144                                 nes_read_1G_phy_reg(nesdev, 0x11,
2145                                                 nesadapter->phy_index[mac_index], &phy_data);
2146                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2147                                                 nesadapter->phy_index[mac_index], phy_data);
2148                                 if (temp_phy_data == phy_data)
2149                                         break;
2150                                 temp_phy_data = phy_data;
2151                         } while (1);
2152
2153                         nes_read_1G_phy_reg(nesdev, 0x1e,
2154                                         nesadapter->phy_index[mac_index], &phy_data);
2155                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2156                                         nesadapter->phy_index[mac_index], phy_data);
2157
2158                         nes_read_1G_phy_reg(nesdev, 1,
2159                                         nesadapter->phy_index[mac_index], &phy_data);
2160                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2161                                         nesadapter->phy_index[mac_index], phy_data);
2162
2163                         if (temp_phy_data & 0x1000) {
2164                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2165                                 phy_data = 4;
2166                         } else {
2167                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2168                         }
2169                 }
2170                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2171                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2172                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2173                 pcs_control_status = nes_read_indexed(nesdev,
2174                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2175                 pcs_control_status = nes_read_indexed(nesdev,
2176                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2177                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2178                                 mac_index, pcs_control_status);
2179                 if (nesadapter->OneG_Mode) {
2180                         u32temp = 0x01010000;
2181                         if (nesadapter->port_count > 2) {
2182                                 u32temp |= 0x02020000;
2183                         }
2184                         if ((pcs_control_status & u32temp)!= u32temp) {
2185                                 phy_data = 0;
2186                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2187                         }
2188                 } else if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
2189                         nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2190                         temp_phy_data = (u16)nes_read_indexed(nesdev,
2191                                                                 NES_IDX_MAC_MDIO_CONTROL);
2192                         u32temp = 20;
2193                         do {
2194                                 nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2195                                 phy_data = (u16)nes_read_indexed(nesdev,
2196                                                                 NES_IDX_MAC_MDIO_CONTROL);
2197                                 if ((phy_data == temp_phy_data) || (!(--u32temp)))
2198                                         break;
2199                                 temp_phy_data = phy_data;
2200                         } while (1);
2201                         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2202                                 __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2203
2204                 } else {
2205                         phy_data = (0x0f0f0000 == (pcs_control_status & 0x0f1f0000)) ? 4 : 0;
2206                 }
2207
2208                 if (phy_data & 0x0004) {
2209                         nesadapter->mac_link_down[mac_index] = 0;
2210                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2211                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2212                                                 nesvnic->linkup);
2213                                 if (nesvnic->linkup == 0) {
2214                                         printk(PFX "The Link is now up for port %u, netdev %p.\n",
2215                                                         mac_index, nesvnic->netdev);
2216                                         if (netif_queue_stopped(nesvnic->netdev))
2217                                                 netif_start_queue(nesvnic->netdev);
2218                                         nesvnic->linkup = 1;
2219                                         netif_carrier_on(nesvnic->netdev);
2220                                 }
2221                         }
2222                 } else {
2223                         nesadapter->mac_link_down[mac_index] = 1;
2224                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2225                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2226                                                 nesvnic->linkup);
2227                                 if (nesvnic->linkup == 1) {
2228                                         printk(PFX "The Link is now down for port %u, netdev %p.\n",
2229                                                         mac_index, nesvnic->netdev);
2230                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2231                                                 netif_stop_queue(nesvnic->netdev);
2232                                         nesvnic->linkup = 0;
2233                                         netif_carrier_off(nesvnic->netdev);
2234                                 }
2235                         }
2236                 }
2237         }
2238
2239         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2240 }
2241
2242
2243
2244 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2245 {
2246         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2247
2248         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2249 }
2250
2251
2252 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2253 * getting out of nic_ce_handler
2254 */
2255 #define MAX_RQES_TO_PROCESS     384
2256
2257 /**
2258  * nes_nic_ce_handler
2259  */
2260 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2261 {
2262         u64 u64temp;
2263         dma_addr_t bus_address;
2264         struct nes_hw_nic *nesnic;
2265         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2266         struct nes_adapter *nesadapter = nesdev->nesadapter;
2267         struct nes_hw_nic_rq_wqe *nic_rqe;
2268         struct nes_hw_nic_sq_wqe *nic_sqe;
2269         struct sk_buff *skb;
2270         struct sk_buff *rx_skb;
2271         __le16 *wqe_fragment_length;
2272         u32 head;
2273         u32 cq_size;
2274         u32 rx_pkt_size;
2275         u32 cqe_count=0;
2276         u32 cqe_errv;
2277         u32 cqe_misc;
2278         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2279         u16 vlan_tag;
2280         u16 pkt_type;
2281         u16 rqes_processed = 0;
2282         u8 sq_cqes = 0;
2283         u8 nes_use_lro = 0;
2284
2285         head = cq->cq_head;
2286         cq_size = cq->cq_size;
2287         cq->cqes_pending = 1;
2288         if (nesvnic->netdev->features & NETIF_F_LRO)
2289                 nes_use_lro = 1;
2290         do {
2291                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2292                                 NES_NIC_CQE_VALID) {
2293                         nesnic = &nesvnic->nic;
2294                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2295                         if (cqe_misc & NES_NIC_CQE_SQ) {
2296                                 sq_cqes++;
2297                                 wqe_fragment_index = 1;
2298                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2299                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2300                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2301                                 /* bump past the vlan tag */
2302                                 wqe_fragment_length++;
2303                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2304                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2305                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2306                                         bus_address = (dma_addr_t)u64temp;
2307                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2308                                                 pci_unmap_single(nesdev->pcidev,
2309                                                                 bus_address,
2310                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2311                                                                 PCI_DMA_TODEVICE);
2312                                         }
2313                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2314                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2315                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2316                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2317                                                         bus_address = (dma_addr_t)u64temp;
2318                                                         pci_unmap_page(nesdev->pcidev,
2319                                                                         bus_address,
2320                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2321                                                                         PCI_DMA_TODEVICE);
2322                                                 } else
2323                                                         break;
2324                                         }
2325                                         if (skb)
2326                                                 dev_kfree_skb_any(skb);
2327                                 }
2328                                 nesnic->sq_tail++;
2329                                 nesnic->sq_tail &= nesnic->sq_size-1;
2330                                 if (sq_cqes > 128) {
2331                                         barrier();
2332                                 /* restart the queue if it had been stopped */
2333                                 if (netif_queue_stopped(nesvnic->netdev))
2334                                         netif_wake_queue(nesvnic->netdev);
2335                                         sq_cqes = 0;
2336                                 }
2337                         } else {
2338                                 rqes_processed ++;
2339
2340                                 cq->rx_cqes_completed++;
2341                                 cq->rx_pkts_indicated++;
2342                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2343                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2344                                 /* Get the skb */
2345                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2346                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2347                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2348                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2349                                 pci_unmap_single(nesdev->pcidev, bus_address,
2350                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2351                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2352                                 /* rx_skb->len = rx_pkt_size; */
2353                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2354                                 skb_put(rx_skb, rx_pkt_size);
2355                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2356                                 nesnic->rq_tail++;
2357                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2358
2359                                 atomic_inc(&nesvnic->rx_skbs_needed);
2360                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2361                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2362                                                         cq->cq_number | (cqe_count << 16));
2363 //                                      nesadapter->tune_timer.cq_count += cqe_count;
2364                                         nesdev->currcq_count += cqe_count;
2365                                         cqe_count = 0;
2366                                         nes_replenish_nic_rq(nesvnic);
2367                                 }
2368                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2369                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2370                                 rx_skb->ip_summed = CHECKSUM_NONE;
2371
2372                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2373                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2374                                         if ((cqe_errv &
2375                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2376                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2377                                                 if (nesvnic->rx_checksum_disabled == 0) {
2378                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2379                                                 }
2380                                         } else
2381                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2382                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2383                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2384
2385                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2386                                         if ((cqe_errv &
2387                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2388                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2389                                                 if (nesvnic->rx_checksum_disabled == 0) {
2390                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2391                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2392                                                                   nesvnic->netdev->name); */
2393                                                 }
2394                                         } else
2395                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2396                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2397                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2398                                         }
2399                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2400                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2401
2402                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2403                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2404                                 } else {
2405                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2406                                                 vlan_tag = (u16)(le32_to_cpu(
2407                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2408                                                                 >> 16);
2409                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2410                                                                 nesvnic->netdev->name, vlan_tag);
2411                                                 if (nes_use_lro)
2412                                                         lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2413                                                                         nesvnic->vlan_grp, vlan_tag, NULL);
2414                                                 else
2415                                                         nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2416                                         } else {
2417                                                 if (nes_use_lro)
2418                                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2419                                                 else
2420                                                         nes_netif_rx(rx_skb);
2421                                         }
2422                                 }
2423
2424                                 nesvnic->netdev->last_rx = jiffies;
2425                                 /* nesvnic->netstats.rx_packets++; */
2426                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2427                         }
2428
2429                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2430                         /* Accounting... */
2431                         cqe_count++;
2432                         if (++head >= cq_size)
2433                                 head = 0;
2434                         if (cqe_count == 255) {
2435                                 /* Replenish Nic CQ */
2436                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2437                                                 cq->cq_number | (cqe_count << 16));
2438 //                              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2439                                 nesdev->currcq_count += cqe_count;
2440                                 cqe_count = 0;
2441                         }
2442
2443                         if (cq->rx_cqes_completed >= nesvnic->budget)
2444                                 break;
2445                 } else {
2446                         cq->cqes_pending = 0;
2447                         break;
2448                 }
2449
2450         } while (1);
2451
2452         if (nes_use_lro)
2453                 lro_flush_all(&nesvnic->lro_mgr);
2454         if (sq_cqes) {
2455                 barrier();
2456                 /* restart the queue if it had been stopped */
2457                 if (netif_queue_stopped(nesvnic->netdev))
2458                         netif_wake_queue(nesvnic->netdev);
2459         }
2460         cq->cq_head = head;
2461         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2462                         cq->cq_number, cqe_count, cq->cq_head); */
2463         cq->cqe_allocs_pending = cqe_count;
2464         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2465         {
2466 //              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2467                 nesdev->currcq_count += cqe_count;
2468                 nes_nic_tune_timer(nesdev);
2469         }
2470         if (atomic_read(&nesvnic->rx_skbs_needed))
2471                 nes_replenish_nic_rq(nesvnic);
2472 }
2473
2474
2475 /**
2476  * nes_cqp_ce_handler
2477  */
2478 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2479 {
2480         u64 u64temp;
2481         unsigned long flags;
2482         struct nes_hw_cqp *cqp = NULL;
2483         struct nes_cqp_request *cqp_request;
2484         struct nes_hw_cqp_wqe *cqp_wqe;
2485         u32 head;
2486         u32 cq_size;
2487         u32 cqe_count=0;
2488         u32 error_code;
2489         /* u32 counter; */
2490
2491         head = cq->cq_head;
2492         cq_size = cq->cq_size;
2493
2494         do {
2495                 /* process the CQE */
2496                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2497                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2498
2499                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2500                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2501                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
2502                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2503                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2504                         cqp = *((struct nes_hw_cqp **)&u64temp);
2505
2506                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2507                         if (error_code) {
2508                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2509                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2510                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2511                                                 (u16)(error_code >> 16),
2512                                                 (u16)error_code);
2513                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2514                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2515                         }
2516
2517                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2518                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX])))<<32) |
2519                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2520                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2521                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2522                         if (cqp_request) {
2523                                 if (cqp_request->waiting) {
2524                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2525                                         cqp_request->major_code = (u16)(error_code >> 16);
2526                                         cqp_request->minor_code = (u16)error_code;
2527                                         barrier();
2528                                         cqp_request->request_done = 1;
2529                                         wake_up(&cqp_request->waitq);
2530                                         if (atomic_dec_and_test(&cqp_request->refcount)) {
2531                                                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2532                                                                 cqp_request,
2533                                                                 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2534                                                 if (cqp_request->dynamic) {
2535                                                         kfree(cqp_request);
2536                                                 } else {
2537                                                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2538                                                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2539                                                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2540                                                 }
2541                                         }
2542                                 } else if (cqp_request->callback) {
2543                                         /* Envoke the callback routine */
2544                                         cqp_request->cqp_callback(nesdev, cqp_request);
2545                                         if (cqp_request->dynamic) {
2546                                                 kfree(cqp_request);
2547                                         } else {
2548                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2549                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2550                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2551                                         }
2552                                 } else {
2553                                         nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2554                                                         cqp_request,
2555                                                         le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2556                                         if (cqp_request->dynamic) {
2557                                                 kfree(cqp_request);
2558                                         } else {
2559                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2560                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2561                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2562                                         }
2563                                 }
2564                         } else {
2565                                 wake_up(&nesdev->cqp.waitq);
2566                         }
2567
2568                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2569                         nes_write32(nesdev->regs+NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2570                         if (++cqp->sq_tail >= cqp->sq_size)
2571                                 cqp->sq_tail = 0;
2572
2573                         /* Accounting... */
2574                         cqe_count++;
2575                         if (++head >= cq_size)
2576                                 head = 0;
2577                 } else {
2578                         break;
2579                 }
2580         } while (1);
2581         cq->cq_head = head;
2582
2583         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2584         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2585                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2586                         (nesdev->cqp.sq_size - 1)) != 1)) {
2587                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2588                                 struct nes_cqp_request, list);
2589                 list_del_init(&cqp_request->list);
2590                 head = nesdev->cqp.sq_head++;
2591                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2592                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2593                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2594                 barrier();
2595                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2596                         cpu_to_le32((u32)((unsigned long)cqp_request));
2597                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2598                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2599                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2600                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2601                 /* Ring doorbell (1 WQEs) */
2602                 barrier();
2603                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2604         }
2605         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2606
2607         /* Arm the CCQ */
2608         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2609                         cq->cq_number);
2610         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2611 }
2612
2613
2614 /**
2615  * nes_process_iwarp_aeqe
2616  */
2617 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2618                                    struct nes_hw_aeqe *aeqe)
2619 {
2620         u64 context;
2621         u64 aeqe_context = 0;
2622         unsigned long flags;
2623         struct nes_qp *nesqp;
2624         int resource_allocated;
2625         /* struct iw_cm_id *cm_id; */
2626         struct nes_adapter *nesadapter = nesdev->nesadapter;
2627         struct ib_event ibevent;
2628         /* struct iw_cm_event cm_event; */
2629         u32 aeq_info;
2630         u32 next_iwarp_state = 0;
2631         u16 async_event_id;
2632         u8 tcp_state;
2633         u8 iwarp_state;
2634
2635         nes_debug(NES_DBG_AEQ, "\n");
2636         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2637         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2638                 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2639                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2640         } else {
2641                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2642                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2643                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2644                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2645                 BUG_ON(!context);
2646         }
2647
2648         async_event_id = (u16)aeq_info;
2649         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2650         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2651         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2652                         " Tcp state = %s, iWARP state = %s\n",
2653                         async_event_id,
2654                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2655                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2656
2657
2658         switch (async_event_id) {
2659                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2660                         nesqp = *((struct nes_qp **)&context);
2661                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2662                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2663                                 nes_add_ref(&nesqp->ibqp);
2664                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2665                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2666                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2667                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2668                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2669                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2670                                                 async_event_id, nesqp->last_aeq, tcp_state);
2671                         }
2672                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2673                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2674                                 /* FIN Received but tcp state or IB state moved on,
2675                                                 should expect a close complete */
2676                                 return;
2677                         }
2678                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2679                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2680                 case NES_AEQE_AEID_TERMINATE_SENT:
2681                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2682                 case NES_AEQE_AEID_RESET_SENT:
2683                         nesqp = *((struct nes_qp **)&context);
2684                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2685                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2686                         }
2687                         nes_add_ref(&nesqp->ibqp);
2688                         spin_lock_irqsave(&nesqp->lock, flags);
2689                         nesqp->hw_iwarp_state = iwarp_state;
2690                         nesqp->hw_tcp_state = tcp_state;
2691                         nesqp->last_aeq = async_event_id;
2692
2693                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2694                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2695                                 nesqp->hte_added = 0;
2696                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2697                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2698                                                 nesqp->hwqp.qp_id);
2699                                 nes_hw_modify_qp(nesdev, nesqp,
2700                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2701                                 spin_lock_irqsave(&nesqp->lock, flags);
2702                         }
2703
2704                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2705                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2706                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2707                                 switch (nesqp->hw_iwarp_state) {
2708                                         case NES_AEQE_IWARP_STATE_RTS:
2709                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2710                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2711                                                 break;
2712                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2713                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2714                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2715                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2716                                                         next_iwarp_state |= 0x02000000;
2717                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2718                                                 }
2719                                                 break;
2720                                         default:
2721                                                 next_iwarp_state = 0;
2722                                 }
2723                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2724                                 if (next_iwarp_state) {
2725                                         nes_add_ref(&nesqp->ibqp);
2726                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2727                                                         " also added another reference\n",
2728                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2729                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2730                                 }
2731                                 nes_cm_disconn(nesqp);
2732                         } else {
2733                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2734                                         /* FIN Received but ib state not RTS,
2735                                                         close complete will be on its way */
2736                                         spin_unlock_irqrestore(&nesqp->lock, flags);
2737                                         nes_rem_ref(&nesqp->ibqp);
2738                                         return;
2739                                 }
2740                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2741                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2742                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2743                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2744                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2745                                                         " also added another reference\n",
2746                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2747                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2748                                 }
2749                                 nes_cm_disconn(nesqp);
2750                         }
2751                         break;
2752                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2753                         nesqp = *((struct nes_qp **)&context);
2754                         spin_lock_irqsave(&nesqp->lock, flags);
2755                         nesqp->hw_iwarp_state = iwarp_state;
2756                         nesqp->hw_tcp_state = tcp_state;
2757                         nesqp->last_aeq = async_event_id;
2758                         spin_unlock_irqrestore(&nesqp->lock, flags);
2759                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2760                                         " event on QP%u \n  Q2 Data:\n",
2761                                         nesqp->hwqp.qp_id);
2762                         if (nesqp->ibqp.event_handler) {
2763                                 ibevent.device = nesqp->ibqp.device;
2764                                 ibevent.element.qp = &nesqp->ibqp;
2765                                 ibevent.event = IB_EVENT_QP_FATAL;
2766                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2767                         }
2768                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2769                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
2770                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2771                                 nes_add_ref(&nesqp->ibqp);
2772                                 nes_cm_disconn(nesqp);
2773                         } else {
2774                                 nesqp->in_disconnect = 0;
2775                                 wake_up(&nesqp->kick_waitq);
2776                         }
2777                         break;
2778                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2779                         nesqp = *((struct nes_qp **)&context);
2780                         nes_add_ref(&nesqp->ibqp);
2781                         spin_lock_irqsave(&nesqp->lock, flags);
2782                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2783                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2784                         nesqp->last_aeq = async_event_id;
2785                         if (nesqp->cm_id) {
2786                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2787                                                 " event on QP%u, remote IP = 0x%08X \n",
2788                                                 nesqp->hwqp.qp_id,
2789                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2790                         } else {
2791                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2792                                                 " event on QP%u \n",
2793                                                 nesqp->hwqp.qp_id);
2794                         }
2795                         spin_unlock_irqrestore(&nesqp->lock, flags);
2796                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2797                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2798                         if (nesqp->ibqp.event_handler) {
2799                                 ibevent.device = nesqp->ibqp.device;
2800                                 ibevent.element.qp = &nesqp->ibqp;
2801                                 ibevent.event = IB_EVENT_QP_FATAL;
2802                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2803                         }
2804                         break;
2805                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2806                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2807                                 nesqp = nesadapter->qp_table[le32_to_cpu(
2808                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2809                         } else {
2810                                 /* TODO: get the actual WQE and mask off wqe index */
2811                                 context &= ~((u64)511);
2812                                 nesqp = *((struct nes_qp **)&context);
2813                         }
2814                         spin_lock_irqsave(&nesqp->lock, flags);
2815                         nesqp->hw_iwarp_state = iwarp_state;
2816                         nesqp->hw_tcp_state = tcp_state;
2817                         nesqp->last_aeq = async_event_id;
2818                         spin_unlock_irqrestore(&nesqp->lock, flags);
2819                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
2820                                         nesqp->hwqp.qp_id);
2821                         if (nesqp->ibqp.event_handler) {
2822                                 ibevent.device = nesqp->ibqp.device;
2823                                 ibevent.element.qp = &nesqp->ibqp;
2824                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2825                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2826                         }
2827                         break;
2828                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
2829                         nesqp = *((struct nes_qp **)&context);
2830                         spin_lock_irqsave(&nesqp->lock, flags);
2831                         nesqp->hw_iwarp_state = iwarp_state;
2832                         nesqp->hw_tcp_state = tcp_state;
2833                         nesqp->last_aeq = async_event_id;
2834                         spin_unlock_irqrestore(&nesqp->lock, flags);
2835                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
2836                                         nesqp->hwqp.qp_id);
2837                         if (nesqp->ibqp.event_handler) {
2838                                 ibevent.device = nesqp->ibqp.device;
2839                                 ibevent.element.qp = &nesqp->ibqp;
2840                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2841                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2842                         }
2843                         break;
2844                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
2845                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
2846                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2847                         spin_lock_irqsave(&nesqp->lock, flags);
2848                         nesqp->hw_iwarp_state = iwarp_state;
2849                         nesqp->hw_tcp_state = tcp_state;
2850                         nesqp->last_aeq = async_event_id;
2851                         spin_unlock_irqrestore(&nesqp->lock, flags);
2852                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
2853                                         " nesqp = %p, AE reported %p\n",
2854                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
2855                         if (nesqp->ibqp.event_handler) {
2856                                 ibevent.device = nesqp->ibqp.device;
2857                                 ibevent.element.qp = &nesqp->ibqp;
2858                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2859                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2860                         }
2861                         break;
2862                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
2863                         context <<= 1;
2864                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
2865                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
2866                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
2867                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2868                         if (resource_allocated) {
2869                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
2870                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2871                         }
2872                         break;
2873                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
2874                         nesqp = nesadapter->qp_table[le32_to_cpu(
2875                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2876                         spin_lock_irqsave(&nesqp->lock, flags);
2877                         nesqp->hw_iwarp_state = iwarp_state;
2878                         nesqp->hw_tcp_state = tcp_state;
2879                         nesqp->last_aeq = async_event_id;
2880                         spin_unlock_irqrestore(&nesqp->lock, flags);
2881                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
2882                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
2883                                         nesqp->hwqp.qp_id);
2884                         if (nesqp->ibqp.event_handler) {
2885                                 ibevent.device = nesqp->ibqp.device;
2886                                 ibevent.element.qp = &nesqp->ibqp;
2887                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2888                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2889                         }
2890                         /* tell cm to disconnect, cm will queue work to thread */
2891                         nes_add_ref(&nesqp->ibqp);
2892                         nes_cm_disconn(nesqp);
2893                         break;
2894                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
2895                         nesqp = *((struct nes_qp **)&context);
2896                         spin_lock_irqsave(&nesqp->lock, flags);
2897                         nesqp->hw_iwarp_state = iwarp_state;
2898                         nesqp->hw_tcp_state = tcp_state;
2899                         nesqp->last_aeq = async_event_id;
2900                         spin_unlock_irqrestore(&nesqp->lock, flags);
2901                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
2902                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
2903                                         nesqp->hwqp.qp_id);
2904                         if (nesqp->ibqp.event_handler) {
2905                                 ibevent.device = nesqp->ibqp.device;
2906                                 ibevent.element.qp = &nesqp->ibqp;
2907                                 ibevent.event = IB_EVENT_QP_FATAL;
2908                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2909                         }
2910                         /* tell cm to disconnect, cm will queue work to thread */
2911                         nes_add_ref(&nesqp->ibqp);
2912                         nes_cm_disconn(nesqp);
2913                         break;
2914                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
2915                         nesqp = *((struct nes_qp **)&context);
2916                         spin_lock_irqsave(&nesqp->lock, flags);
2917                         nesqp->hw_iwarp_state = iwarp_state;
2918                         nesqp->hw_tcp_state = tcp_state;
2919                         nesqp->last_aeq = async_event_id;
2920                         spin_unlock_irqrestore(&nesqp->lock, flags);
2921                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
2922                                         " event on QP%u \n  Q2 Data:\n",
2923                                         nesqp->hwqp.qp_id);
2924                         if (nesqp->ibqp.event_handler) {
2925                                 ibevent.device = nesqp->ibqp.device;
2926                                 ibevent.element.qp = &nesqp->ibqp;
2927                                 ibevent.event = IB_EVENT_QP_FATAL;
2928                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2929                         }
2930                         /* tell cm to disconnect, cm will queue work to thread */
2931                         nes_add_ref(&nesqp->ibqp);
2932                         nes_cm_disconn(nesqp);
2933                         break;
2934                         /* TODO: additional AEs need to be here */
2935                 default:
2936                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
2937                                         async_event_id);
2938                         break;
2939         }
2940
2941 }
2942
2943
2944 /**
2945  * nes_iwarp_ce_handler
2946  */
2947 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
2948 {
2949         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
2950
2951         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
2952                         nescq->hw_cq.cq_number); */
2953         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
2954
2955         if (nescq->ibcq.comp_handler)
2956                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
2957
2958         return;
2959 }
2960
2961
2962 /**
2963  * nes_manage_apbvt()
2964  */
2965 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
2966                 u32 nic_index, u32 add_port)
2967 {
2968         struct nes_device *nesdev = nesvnic->nesdev;
2969         struct nes_hw_cqp_wqe *cqp_wqe;
2970         unsigned long flags;
2971         struct nes_cqp_request *cqp_request;
2972         int ret = 0;
2973         u16 major_code;
2974
2975         /* Send manage APBVT request to CQP */
2976         cqp_request = nes_get_cqp_request(nesdev);
2977         if (cqp_request == NULL) {
2978                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
2979                 return -ENOMEM;
2980         }
2981         cqp_request->waiting = 1;
2982         cqp_wqe = &cqp_request->cqp_wqe;
2983
2984         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
2985                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
2986                         accel_local_port, accel_local_port, nic_index);
2987
2988         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2989         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
2990                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
2991         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2992                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
2993
2994         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
2995
2996         atomic_set(&cqp_request->refcount, 2);
2997         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
2998
2999         if (add_port == NES_MANAGE_APBVT_ADD)
3000                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3001                                 NES_EVENT_TIMEOUT);
3002         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3003                         ret, cqp_request->major_code, cqp_request->minor_code);
3004         major_code = cqp_request->major_code;
3005         if (atomic_dec_and_test(&cqp_request->refcount)) {
3006                 if (cqp_request->dynamic) {
3007                         kfree(cqp_request);
3008                 } else {
3009                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3010                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3011                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3012                 }
3013         }
3014         if (!ret)
3015                 return -ETIME;
3016         else if (major_code)
3017                 return -EIO;
3018         else
3019                 return 0;
3020 }
3021
3022
3023 /**
3024  * nes_manage_arp_cache
3025  */
3026 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3027                 u32 ip_addr, u32 action)
3028 {
3029         struct nes_hw_cqp_wqe *cqp_wqe;
3030         struct nes_vnic *nesvnic = netdev_priv(netdev);
3031         struct nes_device *nesdev;
3032         struct nes_cqp_request *cqp_request;
3033         int arp_index;
3034
3035         nesdev = nesvnic->nesdev;
3036         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3037         if (arp_index == -1) {
3038                 return;
3039         }
3040
3041         /* update the ARP entry */
3042         cqp_request = nes_get_cqp_request(nesdev);
3043         if (cqp_request == NULL) {
3044                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3045                 return;
3046         }
3047         cqp_request->waiting = 0;
3048         cqp_wqe = &cqp_request->cqp_wqe;
3049         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3050
3051         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3052                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3053         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3054                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3055         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3056
3057         if (action == NES_ARP_ADD) {
3058                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3059                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3060                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3061                                 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3062                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3063                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3064         } else {
3065                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3066                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3067         }
3068
3069         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3070                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3071
3072         atomic_set(&cqp_request->refcount, 1);
3073         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3074 }
3075
3076
3077 /**
3078  * flush_wqes
3079  */
3080 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3081                 u32 which_wq, u32 wait_completion)
3082 {
3083         unsigned long flags;
3084         struct nes_cqp_request *cqp_request;
3085         struct nes_hw_cqp_wqe *cqp_wqe;
3086         int ret;
3087
3088         cqp_request = nes_get_cqp_request(nesdev);
3089         if (cqp_request == NULL) {
3090                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3091                 return;
3092         }
3093         if (wait_completion) {
3094                 cqp_request->waiting = 1;
3095                 atomic_set(&cqp_request->refcount, 2);
3096         } else {
3097                 cqp_request->waiting = 0;
3098         }
3099         cqp_wqe = &cqp_request->cqp_wqe;
3100         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3101
3102         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3103                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3104         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3105
3106         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3107
3108         if (wait_completion) {
3109                 /* Wait for CQP */
3110                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3111                                 NES_EVENT_TIMEOUT);
3112                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3113                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3114                                 ret, cqp_request->major_code, cqp_request->minor_code);
3115                 if (atomic_dec_and_test(&cqp_request->refcount)) {
3116                         if (cqp_request->dynamic) {
3117                                 kfree(cqp_request);
3118                         } else {
3119                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3120                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3121                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3122                         }
3123                 }
3124         }
3125 }