SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/net/vmxnet3/vmxnet3_drv.c

   1 /*
   2  * Linux driver for VMware's vmxnet3 ethernet NIC.
   3  *
   4  * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the
   8  * Free Software Foundation; version 2 of the License and no later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  13  * NON INFRINGEMENT. See the GNU General Public License for more
  14  * details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * The full GNU General Public License is included in this distribution in
  21  * the file called "COPYING".
  22  *
  23  * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
  24  *
  25  */
  26
  27 #include "vmxnet3_int.h"
  28
  29 char vmxnet3_driver_name[] = "vmxnet3";
  30 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
  31
  32
  33 /*
  34  * PCI Device ID Table
  35  * Last entry must be all 0s
  36  */
  37 static const struct pci_device_id vmxnet3_pciid_table[] = {
  38         {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
  39         {0}
  40 };
  41
  42 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
  43
  44 static atomic_t devices_found;
  45
  46
  47 /*
  48  *    Enable/Disable the given intr
  49  */
  50 static void
  51 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
  52 {
  53         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
  54 }
  55
  56
  57 static void
  58 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
  59 {
  60         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
  61 }
  62
  63
  64 /*
  65  *    Enable/Disable all intrs used by the device
  66  */
  67 static void
  68 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
  69 {
  70         int i;
  71
  72         for (i = 0; i < adapter->intr.num_intrs; i++)
  73                 vmxnet3_enable_intr(adapter, i);
  74 }
  75
  76
  77 static void
  78 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
  79 {
  80         int i;
  81
  82         for (i = 0; i < adapter->intr.num_intrs; i++)
  83                 vmxnet3_disable_intr(adapter, i);
  84 }
  85
  86
  87 static void
  88 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
  89 {
  90         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
  91 }
  92
  93
  94 static bool
  95 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
  96 {
  97         return netif_queue_stopped(adapter->netdev);
  98 }
  99
 100
 101 static void
 102 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 103 {
 104         tq->stopped = false;
 105         netif_start_queue(adapter->netdev);
 106 }
 107
 108
 109 static void
 110 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 111 {
 112         tq->stopped = false;
 113         netif_wake_queue(adapter->netdev);
 114 }
 115
 116
 117 static void
 118 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 119 {
 120         tq->stopped = true;
 121         tq->num_stop++;
 122         netif_stop_queue(adapter->netdev);
 123 }
 124
 125
 126 /*
 127  * Check the link state. This may start or stop the tx queue.
 128  */
 129 static void
 130 vmxnet3_check_link(struct vmxnet3_adapter *adapter)
 131 {
 132         u32 ret;
 133
 134         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
 135         ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
 136         adapter->link_speed = ret >> 16;
 137         if (ret & 1) { /* Link is up. */
 138                 printk(KERN_INFO "%s: NIC Link is Up %d Mbps\n",
 139                        adapter->netdev->name, adapter->link_speed);
 140                 if (!netif_carrier_ok(adapter->netdev))
 141                         netif_carrier_on(adapter->netdev);
 142
 143                 vmxnet3_tq_start(&adapter->tx_queue, adapter);
 144         } else {
 145                 printk(KERN_INFO "%s: NIC Link is Down\n",
 146                        adapter->netdev->name);
 147                 if (netif_carrier_ok(adapter->netdev))
 148                         netif_carrier_off(adapter->netdev);
 149
 150                 vmxnet3_tq_stop(&adapter->tx_queue, adapter);
 151         }
 152 }
 153
 154
 155 static void
 156 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
 157 {
 158         u32 events = adapter->shared->ecr;
 159         if (!events)
 160                 return;
 161
 162         vmxnet3_ack_events(adapter, events);
 163
 164         /* Check if link state has changed */
 165         if (events & VMXNET3_ECR_LINK)
 166                 vmxnet3_check_link(adapter);
 167
 168         /* Check if there is an error on xmit/recv queues */
 169         if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
 170                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 171                                        VMXNET3_CMD_GET_QUEUE_STATUS);
 172
 173                 if (adapter->tqd_start->status.stopped) {
 174                         printk(KERN_ERR "%s: tq error 0x%x\n",
 175                                adapter->netdev->name,
 176                                adapter->tqd_start->status.error);
 177                 }
 178                 if (adapter->rqd_start->status.stopped) {
 179                         printk(KERN_ERR "%s: rq error 0x%x\n",
 180                                adapter->netdev->name,
 181                                adapter->rqd_start->status.error);
 182                 }
 183
 184                 schedule_work(&adapter->work);
 185         }
 186 }
 187
 188
 189 static void
 190 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
 191                      struct pci_dev *pdev)
 192 {
 193         if (tbi->map_type == VMXNET3_MAP_SINGLE)
 194                 pci_unmap_single(pdev, tbi->dma_addr, tbi->len,
 195                                  PCI_DMA_TODEVICE);
 196         else if (tbi->map_type == VMXNET3_MAP_PAGE)
 197                 pci_unmap_page(pdev, tbi->dma_addr, tbi->len,
 198                                PCI_DMA_TODEVICE);
 199         else
 200                 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
 201
 202         tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
 203 }
 204
 205
 206 static int
 207 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
 208                   struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
 209 {
 210         struct sk_buff *skb;
 211         int entries = 0;
 212
 213         /* no out of order completion */
 214         BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
 215         BUG_ON(tq->tx_ring.base[eop_idx].txd.eop != 1);
 216
 217         skb = tq->buf_info[eop_idx].skb;
 218         BUG_ON(skb == NULL);
 219         tq->buf_info[eop_idx].skb = NULL;
 220
 221         VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
 222
 223         while (tq->tx_ring.next2comp != eop_idx) {
 224                 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
 225                                      pdev);
 226
 227                 /* update next2comp w/o tx_lock. Since we are marking more,
 228                  * instead of less, tx ring entries avail, the worst case is
 229                  * that the tx routine incorrectly re-queues a pkt due to
 230                  * insufficient tx ring entries.
 231                  */
 232                 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
 233                 entries++;
 234         }
 235
 236         dev_kfree_skb_any(skb);
 237         return entries;
 238 }
 239
 240
 241 static int
 242 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
 243                         struct vmxnet3_adapter *adapter)
 244 {
 245         int completed = 0;
 246         union Vmxnet3_GenericDesc *gdesc;
 247
 248         gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
 249         while (gdesc->tcd.gen == tq->comp_ring.gen) {
 250                 completed += vmxnet3_unmap_pkt(gdesc->tcd.txdIdx, tq,
 251                                                adapter->pdev, adapter);
 252
 253                 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
 254                 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
 255         }
 256
 257         if (completed) {
 258                 spin_lock(&tq->tx_lock);
 259                 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
 260                              vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
 261                              VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
 262                              netif_carrier_ok(adapter->netdev))) {
 263                         vmxnet3_tq_wake(tq, adapter);
 264                 }
 265                 spin_unlock(&tq->tx_lock);
 266         }
 267         return completed;
 268 }
 269
 270
 271 static void
 272 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
 273                    struct vmxnet3_adapter *adapter)
 274 {
 275         int i;
 276
 277         while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
 278                 struct vmxnet3_tx_buf_info *tbi;
 279                 union Vmxnet3_GenericDesc *gdesc;
 280
 281                 tbi = tq->buf_info + tq->tx_ring.next2comp;
 282                 gdesc = tq->tx_ring.base + tq->tx_ring.next2comp;
 283
 284                 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
 285                 if (tbi->skb) {
 286                         dev_kfree_skb_any(tbi->skb);
 287                         tbi->skb = NULL;
 288                 }
 289                 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
 290         }
 291
 292         /* sanity check, verify all buffers are indeed unmapped and freed */
 293         for (i = 0; i < tq->tx_ring.size; i++) {
 294                 BUG_ON(tq->buf_info[i].skb != NULL ||
 295                        tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
 296         }
 297
 298         tq->tx_ring.gen = VMXNET3_INIT_GEN;
 299         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
 300
 301         tq->comp_ring.gen = VMXNET3_INIT_GEN;
 302         tq->comp_ring.next2proc = 0;
 303 }
 304
 305
 306 void
 307 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
 308                    struct vmxnet3_adapter *adapter)
 309 {
 310         if (tq->tx_ring.base) {
 311                 pci_free_consistent(adapter->pdev, tq->tx_ring.size *
 312                                     sizeof(struct Vmxnet3_TxDesc),
 313                                     tq->tx_ring.base, tq->tx_ring.basePA);
 314                 tq->tx_ring.base = NULL;
 315         }
 316         if (tq->data_ring.base) {
 317                 pci_free_consistent(adapter->pdev, tq->data_ring.size *
 318                                     sizeof(struct Vmxnet3_TxDataDesc),
 319                                     tq->data_ring.base, tq->data_ring.basePA);
 320                 tq->data_ring.base = NULL;
 321         }
 322         if (tq->comp_ring.base) {
 323                 pci_free_consistent(adapter->pdev, tq->comp_ring.size *
 324                                     sizeof(struct Vmxnet3_TxCompDesc),
 325                                     tq->comp_ring.base, tq->comp_ring.basePA);
 326                 tq->comp_ring.base = NULL;
 327         }
 328         kfree(tq->buf_info);
 329         tq->buf_info = NULL;
 330 }
 331
 332
 333 static void
 334 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
 335                 struct vmxnet3_adapter *adapter)
 336 {
 337         int i;
 338
 339         /* reset the tx ring contents to 0 and reset the tx ring states */
 340         memset(tq->tx_ring.base, 0, tq->tx_ring.size *
 341                sizeof(struct Vmxnet3_TxDesc));
 342         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
 343         tq->tx_ring.gen = VMXNET3_INIT_GEN;
 344
 345         memset(tq->data_ring.base, 0, tq->data_ring.size *
 346                sizeof(struct Vmxnet3_TxDataDesc));
 347
 348         /* reset the tx comp ring contents to 0 and reset comp ring states */
 349         memset(tq->comp_ring.base, 0, tq->comp_ring.size *
 350                sizeof(struct Vmxnet3_TxCompDesc));
 351         tq->comp_ring.next2proc = 0;
 352         tq->comp_ring.gen = VMXNET3_INIT_GEN;
 353
 354         /* reset the bookkeeping data */
 355         memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
 356         for (i = 0; i < tq->tx_ring.size; i++)
 357                 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
 358
 359         /* stats are not reset */
 360 }
 361
 362
 363 static int
 364 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
 365                   struct vmxnet3_adapter *adapter)
 366 {
 367         BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
 368                tq->comp_ring.base || tq->buf_info);
 369
 370         tq->tx_ring.base = pci_alloc_consistent(adapter->pdev, tq->tx_ring.size
 371                            * sizeof(struct Vmxnet3_TxDesc),
 372                            &tq->tx_ring.basePA);
 373         if (!tq->tx_ring.base) {
 374                 printk(KERN_ERR "%s: failed to allocate tx ring\n",
 375                        adapter->netdev->name);
 376                 goto err;
 377         }
 378
 379         tq->data_ring.base = pci_alloc_consistent(adapter->pdev,
 380                              tq->data_ring.size *
 381                              sizeof(struct Vmxnet3_TxDataDesc),
 382                              &tq->data_ring.basePA);
 383         if (!tq->data_ring.base) {
 384                 printk(KERN_ERR "%s: failed to allocate data ring\n",
 385                        adapter->netdev->name);
 386                 goto err;
 387         }
 388
 389         tq->comp_ring.base = pci_alloc_consistent(adapter->pdev,
 390                              tq->comp_ring.size *
 391                              sizeof(struct Vmxnet3_TxCompDesc),
 392                              &tq->comp_ring.basePA);
 393         if (!tq->comp_ring.base) {
 394                 printk(KERN_ERR "%s: failed to allocate tx comp ring\n",
 395                        adapter->netdev->name);
 396                 goto err;
 397         }
 398
 399         tq->buf_info = kcalloc(tq->tx_ring.size, sizeof(tq->buf_info[0]),
 400                                GFP_KERNEL);
 401         if (!tq->buf_info) {
 402                 printk(KERN_ERR "%s: failed to allocate tx bufinfo\n",
 403                        adapter->netdev->name);
 404                 goto err;
 405         }
 406
 407         return 0;
 408
 409 err:
 410         vmxnet3_tq_destroy(tq, adapter);
 411         return -ENOMEM;
 412 }
 413
 414
 415 /*
 416  *    starting from ring->next2fill, allocate rx buffers for the given ring
 417  *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
 418  *    are allocated or allocation fails
 419  */
 420
 421 static int
 422 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
 423                         int num_to_alloc, struct vmxnet3_adapter *adapter)
 424 {
 425         int num_allocated = 0;
 426         struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
 427         struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
 428         u32 val;
 429
 430         while (num_allocated < num_to_alloc) {
 431                 struct vmxnet3_rx_buf_info *rbi;
 432                 union Vmxnet3_GenericDesc *gd;
 433
 434                 rbi = rbi_base + ring->next2fill;
 435                 gd = ring->base + ring->next2fill;
 436
 437                 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
 438                         if (rbi->skb == NULL) {
 439                                 rbi->skb = dev_alloc_skb(rbi->len +
 440                                                          NET_IP_ALIGN);
 441                                 if (unlikely(rbi->skb == NULL)) {
 442                                         rq->stats.rx_buf_alloc_failure++;
 443                                         break;
 444                                 }
 445                                 rbi->skb->dev = adapter->netdev;
 446
 447                                 skb_reserve(rbi->skb, NET_IP_ALIGN);
 448                                 rbi->dma_addr = pci_map_single(adapter->pdev,
 449                                                 rbi->skb->data, rbi->len,
 450                                                 PCI_DMA_FROMDEVICE);
 451                         } else {
 452                                 /* rx buffer skipped by the device */
 453                         }
 454                         val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
 455                 } else {
 456                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
 457                                rbi->len  != PAGE_SIZE);
 458
 459                         if (rbi->page == NULL) {
 460                                 rbi->page = alloc_page(GFP_ATOMIC);
 461                                 if (unlikely(rbi->page == NULL)) {
 462                                         rq->stats.rx_buf_alloc_failure++;
 463                                         break;
 464                                 }
 465                                 rbi->dma_addr = pci_map_page(adapter->pdev,
 466                                                 rbi->page, 0, PAGE_SIZE,
 467                                                 PCI_DMA_FROMDEVICE);
 468                         } else {
 469                                 /* rx buffers skipped by the device */
 470                         }
 471                         val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
 472                 }
 473
 474                 BUG_ON(rbi->dma_addr == 0);
 475                 gd->rxd.addr = rbi->dma_addr;
 476                 gd->dword[2] = (ring->gen << VMXNET3_RXD_GEN_SHIFT) | val |
 477                                 rbi->len;
 478
 479                 num_allocated++;
 480                 vmxnet3_cmd_ring_adv_next2fill(ring);
 481         }
 482         rq->uncommitted[ring_idx] += num_allocated;
 483
 484         dprintk(KERN_ERR "alloc_rx_buf: %d allocated, next2fill %u, next2comp "
 485                 "%u, uncommited %u\n", num_allocated, ring->next2fill,
 486                 ring->next2comp, rq->uncommitted[ring_idx]);
 487
 488         /* so that the device can distinguish a full ring and an empty ring */
 489         BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
 490
 491         return num_allocated;
 492 }
 493
 494
 495 static void
 496 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
 497                     struct vmxnet3_rx_buf_info *rbi)
 498 {
 499         struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
 500                 skb_shinfo(skb)->nr_frags;
 501
 502         BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
 503
 504         frag->page = rbi->page;
 505         frag->page_offset = 0;
 506         frag->size = rcd->len;
 507         skb->data_len += frag->size;
 508         skb_shinfo(skb)->nr_frags++;
 509 }
 510
 511
 512 static void
 513 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
 514                 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
 515                 struct vmxnet3_adapter *adapter)
 516 {
 517         u32 dw2, len;
 518         unsigned long buf_offset;
 519         int i;
 520         union Vmxnet3_GenericDesc *gdesc;
 521         struct vmxnet3_tx_buf_info *tbi = NULL;
 522
 523         BUG_ON(ctx->copy_size > skb_headlen(skb));
 524
 525         /* use the previous gen bit for the SOP desc */
 526         dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 527
 528         ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
 529         gdesc = ctx->sop_txd; /* both loops below can be skipped */
 530
 531         /* no need to map the buffer if headers are copied */
 532         if (ctx->copy_size) {
 533                 ctx->sop_txd->txd.addr = tq->data_ring.basePA +
 534                                         tq->tx_ring.next2fill *
 535                                         sizeof(struct Vmxnet3_TxDataDesc);
 536                 ctx->sop_txd->dword[2] = dw2 | ctx->copy_size;
 537                 ctx->sop_txd->dword[3] = 0;
 538
 539                 tbi = tq->buf_info + tq->tx_ring.next2fill;
 540                 tbi->map_type = VMXNET3_MAP_NONE;
 541
 542                 dprintk(KERN_ERR "txd[%u]: 0x%Lx 0x%x 0x%x\n",
 543                         tq->tx_ring.next2fill, ctx->sop_txd->txd.addr,
 544                         ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
 545                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 546
 547                 /* use the right gen for non-SOP desc */
 548                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 549         }
 550
 551         /* linear part can use multiple tx desc if it's big */
 552         len = skb_headlen(skb) - ctx->copy_size;
 553         buf_offset = ctx->copy_size;
 554         while (len) {
 555                 u32 buf_size;
 556
 557                 buf_size = len > VMXNET3_MAX_TX_BUF_SIZE ?
 558                            VMXNET3_MAX_TX_BUF_SIZE : len;
 559
 560                 tbi = tq->buf_info + tq->tx_ring.next2fill;
 561                 tbi->map_type = VMXNET3_MAP_SINGLE;
 562                 tbi->dma_addr = pci_map_single(adapter->pdev,
 563                                 skb->data + buf_offset, buf_size,
 564                                 PCI_DMA_TODEVICE);
 565
 566                 tbi->len = buf_size; /* this automatically convert 2^14 to 0 */
 567
 568                 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
 569                 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
 570
 571                 gdesc->txd.addr = tbi->dma_addr;
 572                 gdesc->dword[2] = dw2 | buf_size;
 573                 gdesc->dword[3] = 0;
 574
 575                 dprintk(KERN_ERR "txd[%u]: 0x%Lx 0x%x 0x%x\n",
 576                         tq->tx_ring.next2fill, gdesc->txd.addr,
 577                         gdesc->dword[2], gdesc->dword[3]);
 578                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 579                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 580
 581                 len -= buf_size;
 582                 buf_offset += buf_size;
 583         }
 584
 585         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 586                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 587
 588                 tbi = tq->buf_info + tq->tx_ring.next2fill;
 589                 tbi->map_type = VMXNET3_MAP_PAGE;
 590                 tbi->dma_addr = pci_map_page(adapter->pdev, frag->page,
 591                                              frag->page_offset, frag->size,
 592                                              PCI_DMA_TODEVICE);
 593
 594                 tbi->len = frag->size;
 595
 596                 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
 597                 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
 598
 599                 gdesc->txd.addr = tbi->dma_addr;
 600                 gdesc->dword[2] = dw2 | frag->size;
 601                 gdesc->dword[3] = 0;
 602
 603                 dprintk(KERN_ERR "txd[%u]: 0x%llu %u %u\n",
 604                         tq->tx_ring.next2fill, gdesc->txd.addr,
 605                         gdesc->dword[2], gdesc->dword[3]);
 606                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 607                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 608         }
 609
 610         ctx->eop_txd = gdesc;
 611
 612         /* set the last buf_info for the pkt */
 613         tbi->skb = skb;
 614         tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
 615 }
 616
 617
 618 /*
 619  *    parse and copy relevant protocol headers:
 620  *      For a tso pkt, relevant headers are L2/3/4 including options
 621  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
 622  *      if it's a TCP/UDP pkt
 623  *
 624  * Returns:
 625  *    -1:  error happens during parsing
 626  *     0:  protocol headers parsed, but too big to be copied
 627  *     1:  protocol headers parsed and copied
 628  *
 629  * Other effects:
 630  *    1. related *ctx fields are updated.
 631  *    2. ctx->copy_size is # of bytes copied
 632  *    3. the portion copied is guaranteed to be in the linear part
 633  *
 634  */
 635 static int
 636 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 637                            struct vmxnet3_tx_ctx *ctx,
 638                            struct vmxnet3_adapter *adapter)
 639 {
 640         struct Vmxnet3_TxDataDesc *tdd;
 641
 642         if (ctx->mss) {
 643                 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
 644                 ctx->l4_hdr_size = ((struct tcphdr *)
 645                                    skb_transport_header(skb))->doff * 4;
 646                 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
 647         } else {
 648                 unsigned int pull_size;
 649
 650                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 651                         ctx->eth_ip_hdr_size = skb_transport_offset(skb);
 652
 653                         if (ctx->ipv4) {
 654                                 struct iphdr *iph = (struct iphdr *)
 655                                                     skb_network_header(skb);
 656                                 if (iph->protocol == IPPROTO_TCP) {
 657                                         pull_size = ctx->eth_ip_hdr_size +
 658                                                     sizeof(struct tcphdr);
 659
 660                                         if (unlikely(!pskb_may_pull(skb,
 661                                                                 pull_size))) {
 662                                                 goto err;
 663                                         }
 664                                         ctx->l4_hdr_size = ((struct tcphdr *)
 665                                            skb_transport_header(skb))->doff * 4;
 666                                 } else if (iph->protocol == IPPROTO_UDP) {
 667                                         ctx->l4_hdr_size =
 668                                                         sizeof(struct udphdr);
 669                                 } else {
 670                                         ctx->l4_hdr_size = 0;
 671                                 }
 672                         } else {
 673                                 /* for simplicity, don't copy L4 headers */
 674                                 ctx->l4_hdr_size = 0;
 675                         }
 676                         ctx->copy_size = ctx->eth_ip_hdr_size +
 677                                          ctx->l4_hdr_size;
 678                 } else {
 679                         ctx->eth_ip_hdr_size = 0;
 680                         ctx->l4_hdr_size = 0;
 681                         /* copy as much as allowed */
 682                         ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
 683                                              , skb_headlen(skb));
 684                 }
 685
 686                 /* make sure headers are accessible directly */
 687                 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
 688                         goto err;
 689         }
 690
 691         if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
 692                 tq->stats.oversized_hdr++;
 693                 ctx->copy_size = 0;
 694                 return 0;
 695         }
 696
 697         tdd = tq->data_ring.base + tq->tx_ring.next2fill;
 698
 699         memcpy(tdd->data, skb->data, ctx->copy_size);
 700         dprintk(KERN_ERR "copy %u bytes to dataRing[%u]\n",
 701                 ctx->copy_size, tq->tx_ring.next2fill);
 702         return 1;
 703
 704 err:
 705         return -1;
 706 }
 707
 708
 709 static void
 710 vmxnet3_prepare_tso(struct sk_buff *skb,
 711                     struct vmxnet3_tx_ctx *ctx)
 712 {
 713         struct tcphdr *tcph = (struct tcphdr *)skb_transport_header(skb);
 714         if (ctx->ipv4) {
 715                 struct iphdr *iph = (struct iphdr *)skb_network_header(skb);
 716                 iph->check = 0;
 717                 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
 718                                                  IPPROTO_TCP, 0);
 719         } else {
 720                 struct ipv6hdr *iph = (struct ipv6hdr *)skb_network_header(skb);
 721                 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
 722                                                IPPROTO_TCP, 0);
 723         }
 724 }
 725
 726
 727 /*
 728  * Transmits a pkt thru a given tq
 729  * Returns:
 730  *    NETDEV_TX_OK:      descriptors are setup successfully
 731  *    NETDEV_TX_OK:      error occured, the pkt is dropped
 732  *    NETDEV_TX_BUSY:    tx ring is full, queue is stopped
 733  *
 734  * Side-effects:
 735  *    1. tx ring may be changed
 736  *    2. tq stats may be updated accordingly
 737  *    3. shared->txNumDeferred may be updated
 738  */
 739
 740 static int
 741 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 742                 struct vmxnet3_adapter *adapter, struct net_device *netdev)
 743 {
 744         int ret;
 745         u32 count;
 746         unsigned long flags;
 747         struct vmxnet3_tx_ctx ctx;
 748         union Vmxnet3_GenericDesc *gdesc;
 749
 750         /* conservatively estimate # of descriptors to use */
 751         count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) +
 752                 skb_shinfo(skb)->nr_frags + 1;
 753
 754         ctx.ipv4 = (skb->protocol == __constant_ntohs(ETH_P_IP));
 755
 756         ctx.mss = skb_shinfo(skb)->gso_size;
 757         if (ctx.mss) {
 758                 if (skb_header_cloned(skb)) {
 759                         if (unlikely(pskb_expand_head(skb, 0, 0,
 760                                                       GFP_ATOMIC) != 0)) {
 761                                 tq->stats.drop_tso++;
 762                                 goto drop_pkt;
 763                         }
 764                         tq->stats.copy_skb_header++;
 765                 }
 766                 vmxnet3_prepare_tso(skb, &ctx);
 767         } else {
 768                 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
 769
 770                         /* non-tso pkts must not use more than
 771                          * VMXNET3_MAX_TXD_PER_PKT entries
 772                          */
 773                         if (skb_linearize(skb) != 0) {
 774                                 tq->stats.drop_too_many_frags++;
 775                                 goto drop_pkt;
 776                         }
 777                         tq->stats.linearized++;
 778
 779                         /* recalculate the # of descriptors to use */
 780                         count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
 781                 }
 782         }
 783
 784         ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
 785         if (ret >= 0) {
 786                 BUG_ON(ret <= 0 && ctx.copy_size != 0);
 787                 /* hdrs parsed, check against other limits */
 788                 if (ctx.mss) {
 789                         if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
 790                                      VMXNET3_MAX_TX_BUF_SIZE)) {
 791                                 goto hdr_too_big;
 792                         }
 793                 } else {
 794                         if (skb->ip_summed == CHECKSUM_PARTIAL) {
 795                                 if (unlikely(ctx.eth_ip_hdr_size +
 796                                              skb->csum_offset >
 797                                              VMXNET3_MAX_CSUM_OFFSET)) {
 798                                         goto hdr_too_big;
 799                                 }
 800                         }
 801                 }
 802         } else {
 803                 tq->stats.drop_hdr_inspect_err++;
 804                 goto drop_pkt;
 805         }
 806
 807         spin_lock_irqsave(&tq->tx_lock, flags);
 808
 809         if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
 810                 tq->stats.tx_ring_full++;
 811                 dprintk(KERN_ERR "tx queue stopped on %s, next2comp %u"
 812                         " next2fill %u\n", adapter->netdev->name,
 813                         tq->tx_ring.next2comp, tq->tx_ring.next2fill);
 814
 815                 vmxnet3_tq_stop(tq, adapter);
 816                 spin_unlock_irqrestore(&tq->tx_lock, flags);
 817                 return NETDEV_TX_BUSY;
 818         }
 819
 820         /* fill tx descs related to addr & len */
 821         vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
 822
 823         /* setup the EOP desc */
 824         ctx.eop_txd->dword[3] = VMXNET3_TXD_CQ | VMXNET3_TXD_EOP;
 825
 826         /* setup the SOP desc */
 827         gdesc = ctx.sop_txd;
 828         if (ctx.mss) {
 829                 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
 830                 gdesc->txd.om = VMXNET3_OM_TSO;
 831                 gdesc->txd.msscof = ctx.mss;
 832                 tq->shared->txNumDeferred += (skb->len - gdesc->txd.hlen +
 833                                              ctx.mss - 1) / ctx.mss;
 834         } else {
 835                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 836                         gdesc->txd.hlen = ctx.eth_ip_hdr_size;
 837                         gdesc->txd.om = VMXNET3_OM_CSUM;
 838                         gdesc->txd.msscof = ctx.eth_ip_hdr_size +
 839                                             skb->csum_offset;
 840                 } else {
 841                         gdesc->txd.om = 0;
 842                         gdesc->txd.msscof = 0;
 843                 }
 844                 tq->shared->txNumDeferred++;
 845         }
 846
 847         if (vlan_tx_tag_present(skb)) {
 848                 gdesc->txd.ti = 1;
 849                 gdesc->txd.tci = vlan_tx_tag_get(skb);
 850         }
 851
 852         wmb();
 853
 854         /* finally flips the GEN bit of the SOP desc */
 855         gdesc->dword[2] ^= VMXNET3_TXD_GEN;
 856         dprintk(KERN_ERR "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
 857                 (u32)((union Vmxnet3_GenericDesc *)ctx.sop_txd -
 858                 tq->tx_ring.base), gdesc->txd.addr, gdesc->dword[2],
 859                 gdesc->dword[3]);
 860
 861         spin_unlock_irqrestore(&tq->tx_lock, flags);
 862
 863         if (tq->shared->txNumDeferred >= tq->shared->txThreshold) {
 864                 tq->shared->txNumDeferred = 0;
 865                 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_TXPROD,
 866                                        tq->tx_ring.next2fill);
 867         }
 868         netdev->trans_start = jiffies;
 869
 870         return NETDEV_TX_OK;
 871
 872 hdr_too_big:
 873         tq->stats.drop_oversized_hdr++;
 874 drop_pkt:
 875         tq->stats.drop_total++;
 876         dev_kfree_skb(skb);
 877         return NETDEV_TX_OK;
 878 }
 879
 880
 881 static netdev_tx_t
 882 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 883 {
 884         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 885         struct vmxnet3_tx_queue *tq = &adapter->tx_queue;
 886
 887         return vmxnet3_tq_xmit(skb, tq, adapter, netdev);
 888 }
 889
 890
 891 static void
 892 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
 893                 struct sk_buff *skb,
 894                 union Vmxnet3_GenericDesc *gdesc)
 895 {
 896         if (!gdesc->rcd.cnc && adapter->rxcsum) {
 897                 /* typical case: TCP/UDP over IP and both csums are correct */
 898                 if ((gdesc->dword[3] & VMXNET3_RCD_CSUM_OK) ==
 899                                                         VMXNET3_RCD_CSUM_OK) {
 900                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 901                         BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
 902                         BUG_ON(!(gdesc->rcd.v4  || gdesc->rcd.v6));
 903                         BUG_ON(gdesc->rcd.frg);
 904                 } else {
 905                         if (gdesc->rcd.csum) {
 906                                 skb->csum = htons(gdesc->rcd.csum);
 907                                 skb->ip_summed = CHECKSUM_PARTIAL;
 908                         } else {
 909                                 skb->ip_summed = CHECKSUM_NONE;
 910                         }
 911                 }
 912         } else {
 913                 skb->ip_summed = CHECKSUM_NONE;
 914         }
 915 }
 916
 917
 918 static void
 919 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
 920                  struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
 921 {
 922         rq->stats.drop_err++;
 923         if (!rcd->fcs)
 924                 rq->stats.drop_fcs++;
 925
 926         rq->stats.drop_total++;
 927
 928         /*
 929          * We do not unmap and chain the rx buffer to the skb.
 930          * We basically pretend this buffer is not used and will be recycled
 931          * by vmxnet3_rq_alloc_rx_buf()
 932          */
 933
 934         /*
 935          * ctx->skb may be NULL if this is the first and the only one
 936          * desc for the pkt
 937          */
 938         if (ctx->skb)
 939                 dev_kfree_skb_irq(ctx->skb);
 940
 941         ctx->skb = NULL;
 942 }
 943
 944
 945 static int
 946 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
 947                        struct vmxnet3_adapter *adapter, int quota)
 948 {
 949         static u32 rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
 950         u32 num_rxd = 0;
 951         struct Vmxnet3_RxCompDesc *rcd;
 952         struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
 953
 954         rcd = &rq->comp_ring.base[rq->comp_ring.next2proc].rcd;
 955         while (rcd->gen == rq->comp_ring.gen) {
 956                 struct vmxnet3_rx_buf_info *rbi;
 957                 struct sk_buff *skb;
 958                 int num_to_alloc;
 959                 struct Vmxnet3_RxDesc *rxd;
 960                 u32 idx, ring_idx;
 961
 962                 if (num_rxd >= quota) {
 963                         /* we may stop even before we see the EOP desc of
 964                          * the current pkt
 965                          */
 966                         break;
 967                 }
 968                 num_rxd++;
 969
 970                 idx = rcd->rxdIdx;
 971                 ring_idx = rcd->rqID == rq->qid ? 0 : 1;
 972
 973                 rxd = &rq->rx_ring[ring_idx].base[idx].rxd;
 974                 rbi = rq->buf_info[ring_idx] + idx;
 975
 976                 BUG_ON(rxd->addr != rbi->dma_addr || rxd->len != rbi->len);
 977
 978                 if (unlikely(rcd->eop && rcd->err)) {
 979                         vmxnet3_rx_error(rq, rcd, ctx, adapter);
 980                         goto rcd_done;
 981                 }
 982
 983                 if (rcd->sop) { /* first buf of the pkt */
 984                         BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
 985                                rcd->rqID != rq->qid);
 986
 987                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
 988                         BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
 989
 990                         if (unlikely(rcd->len == 0)) {
 991                                 /* Pretend the rx buffer is skipped. */
 992                                 BUG_ON(!(rcd->sop && rcd->eop));
 993                                 dprintk(KERN_ERR "rxRing[%u][%u] 0 length\n",
 994                                         ring_idx, idx);
 995                                 goto rcd_done;
 996                         }
 997
 998                         ctx->skb = rbi->skb;
 999                         rbi->skb = NULL;
1000
1001                         pci_unmap_single(adapter->pdev, rbi->dma_addr, rbi->len,
1002                                          PCI_DMA_FROMDEVICE);
1003
1004                         skb_put(ctx->skb, rcd->len);
1005                 } else {
1006                         BUG_ON(ctx->skb == NULL);
1007                         /* non SOP buffer must be type 1 in most cases */
1008                         if (rbi->buf_type == VMXNET3_RX_BUF_PAGE) {
1009                                 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1010
1011                                 if (rcd->len) {
1012                                         pci_unmap_page(adapter->pdev,
1013                                                        rbi->dma_addr, rbi->len,
1014                                                        PCI_DMA_FROMDEVICE);
1015
1016                                         vmxnet3_append_frag(ctx->skb, rcd, rbi);
1017                                         rbi->page = NULL;
1018                                 }
1019                         } else {
1020                                 /*
1021                                  * The only time a non-SOP buffer is type 0 is
1022                                  * when it's EOP and error flag is raised, which
1023                                  * has already been handled.
1024                                  */
1025                                 BUG_ON(true);
1026                         }
1027                 }
1028
1029                 skb = ctx->skb;
1030                 if (rcd->eop) {
1031                         skb->len += skb->data_len;
1032                         skb->truesize += skb->data_len;
1033
1034                         vmxnet3_rx_csum(adapter, skb,
1035                                         (union Vmxnet3_GenericDesc *)rcd);
1036                         skb->protocol = eth_type_trans(skb, adapter->netdev);
1037
1038                         if (unlikely(adapter->vlan_grp && rcd->ts)) {
1039                                 vlan_hwaccel_receive_skb(skb,
1040                                                 adapter->vlan_grp, rcd->tci);
1041                         } else {
1042                                 netif_receive_skb(skb);
1043                         }
1044
1045                         adapter->netdev->last_rx = jiffies;
1046                         ctx->skb = NULL;
1047                 }
1048
1049 rcd_done:
1050                 /* device may skip some rx descs */
1051                 rq->rx_ring[ring_idx].next2comp = idx;
1052                 VMXNET3_INC_RING_IDX_ONLY(rq->rx_ring[ring_idx].next2comp,
1053                                           rq->rx_ring[ring_idx].size);
1054
1055                 /* refill rx buffers frequently to avoid starving the h/w */
1056                 num_to_alloc = vmxnet3_cmd_ring_desc_avail(rq->rx_ring +
1057                                                            ring_idx);
1058                 if (unlikely(num_to_alloc > VMXNET3_RX_ALLOC_THRESHOLD(rq,
1059                                                         ring_idx, adapter))) {
1060                         vmxnet3_rq_alloc_rx_buf(rq, ring_idx, num_to_alloc,
1061                                                 adapter);
1062
1063                         /* if needed, update the register */
1064                         if (unlikely(rq->shared->updateRxProd)) {
1065                                 VMXNET3_WRITE_BAR0_REG(adapter,
1066                                         rxprod_reg[ring_idx] + rq->qid * 8,
1067                                         rq->rx_ring[ring_idx].next2fill);
1068                                 rq->uncommitted[ring_idx] = 0;
1069                         }
1070                 }
1071
1072                 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1073                 rcd = &rq->comp_ring.base[rq->comp_ring.next2proc].rcd;
1074         }
1075
1076         return num_rxd;
1077 }
1078
1079
1080 static void
1081 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1082                    struct vmxnet3_adapter *adapter)
1083 {
1084         u32 i, ring_idx;
1085         struct Vmxnet3_RxDesc *rxd;
1086
1087         for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1088                 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1089                         rxd = &rq->rx_ring[ring_idx].base[i].rxd;
1090
1091                         if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1092                                         rq->buf_info[ring_idx][i].skb) {
1093                                 pci_unmap_single(adapter->pdev, rxd->addr,
1094                                                  rxd->len, PCI_DMA_FROMDEVICE);
1095                                 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1096                                 rq->buf_info[ring_idx][i].skb = NULL;
1097                         } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1098                                         rq->buf_info[ring_idx][i].page) {
1099                                 pci_unmap_page(adapter->pdev, rxd->addr,
1100                                                rxd->len, PCI_DMA_FROMDEVICE);
1101                                 put_page(rq->buf_info[ring_idx][i].page);
1102                                 rq->buf_info[ring_idx][i].page = NULL;
1103                         }
1104                 }
1105
1106                 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1107                 rq->rx_ring[ring_idx].next2fill =
1108                                         rq->rx_ring[ring_idx].next2comp = 0;
1109                 rq->uncommitted[ring_idx] = 0;
1110         }
1111
1112         rq->comp_ring.gen = VMXNET3_INIT_GEN;
1113         rq->comp_ring.next2proc = 0;
1114 }
1115
1116
1117 void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1118                         struct vmxnet3_adapter *adapter)
1119 {
1120         int i;
1121         int j;
1122
1123         /* all rx buffers must have already been freed */
1124         for (i = 0; i < 2; i++) {
1125                 if (rq->buf_info[i]) {
1126                         for (j = 0; j < rq->rx_ring[i].size; j++)
1127                                 BUG_ON(rq->buf_info[i][j].page != NULL);
1128                 }
1129         }
1130
1131
1132         kfree(rq->buf_info[0]);
1133
1134         for (i = 0; i < 2; i++) {
1135                 if (rq->rx_ring[i].base) {
1136                         pci_free_consistent(adapter->pdev, rq->rx_ring[i].size
1137                                             * sizeof(struct Vmxnet3_RxDesc),
1138                                             rq->rx_ring[i].base,
1139                                             rq->rx_ring[i].basePA);
1140                         rq->rx_ring[i].base = NULL;
1141                 }
1142                 rq->buf_info[i] = NULL;
1143         }
1144
1145         if (rq->comp_ring.base) {
1146                 pci_free_consistent(adapter->pdev, rq->comp_ring.size *
1147                                     sizeof(struct Vmxnet3_RxCompDesc),
1148                                     rq->comp_ring.base, rq->comp_ring.basePA);
1149                 rq->comp_ring.base = NULL;
1150         }
1151 }
1152
1153
1154 static int
1155 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1156                 struct vmxnet3_adapter  *adapter)
1157 {
1158         int i;
1159
1160         /* initialize buf_info */
1161         for (i = 0; i < rq->rx_ring[0].size; i++) {
1162
1163                 /* 1st buf for a pkt is skbuff */
1164                 if (i % adapter->rx_buf_per_pkt == 0) {
1165                         rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1166                         rq->buf_info[0][i].len = adapter->skb_buf_size;
1167                 } else { /* subsequent bufs for a pkt is frag */
1168                         rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1169                         rq->buf_info[0][i].len = PAGE_SIZE;
1170                 }
1171         }
1172         for (i = 0; i < rq->rx_ring[1].size; i++) {
1173                 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1174                 rq->buf_info[1][i].len = PAGE_SIZE;
1175         }
1176
1177         /* reset internal state and allocate buffers for both rings */
1178         for (i = 0; i < 2; i++) {
1179                 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1180                 rq->uncommitted[i] = 0;
1181
1182                 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1183                        sizeof(struct Vmxnet3_RxDesc));
1184                 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1185         }
1186         if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1187                                     adapter) == 0) {
1188                 /* at least has 1 rx buffer for the 1st ring */
1189                 return -ENOMEM;
1190         }
1191         vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1192
1193         /* reset the comp ring */
1194         rq->comp_ring.next2proc = 0;
1195         memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1196                sizeof(struct Vmxnet3_RxCompDesc));
1197         rq->comp_ring.gen = VMXNET3_INIT_GEN;
1198
1199         /* reset rxctx */
1200         rq->rx_ctx.skb = NULL;
1201
1202         /* stats are not reset */
1203         return 0;
1204 }
1205
1206
1207 static int
1208 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1209 {
1210         int i;
1211         size_t sz;
1212         struct vmxnet3_rx_buf_info *bi;
1213
1214         for (i = 0; i < 2; i++) {
1215
1216                 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1217                 rq->rx_ring[i].base = pci_alloc_consistent(adapter->pdev, sz,
1218                                                         &rq->rx_ring[i].basePA);
1219                 if (!rq->rx_ring[i].base) {
1220                         printk(KERN_ERR "%s: failed to allocate rx ring %d\n",
1221                                adapter->netdev->name, i);
1222                         goto err;
1223                 }
1224         }
1225
1226         sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1227         rq->comp_ring.base = pci_alloc_consistent(adapter->pdev, sz,
1228                                                   &rq->comp_ring.basePA);
1229         if (!rq->comp_ring.base) {
1230                 printk(KERN_ERR "%s: failed to allocate rx comp ring\n",
1231                        adapter->netdev->name);
1232                 goto err;
1233         }
1234
1235         sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1236                                                    rq->rx_ring[1].size);
1237         bi = kmalloc(sz, GFP_KERNEL);
1238         if (!bi) {
1239                 printk(KERN_ERR "%s: failed to allocate rx bufinfo\n",
1240                        adapter->netdev->name);
1241                 goto err;
1242         }
1243         memset(bi, 0, sz);
1244         rq->buf_info[0] = bi;
1245         rq->buf_info[1] = bi + rq->rx_ring[0].size;
1246
1247         return 0;
1248
1249 err:
1250         vmxnet3_rq_destroy(rq, adapter);
1251         return -ENOMEM;
1252 }
1253
1254
1255 static int
1256 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1257 {
1258         if (unlikely(adapter->shared->ecr))
1259                 vmxnet3_process_events(adapter);
1260
1261         vmxnet3_tq_tx_complete(&adapter->tx_queue, adapter);
1262         return vmxnet3_rq_rx_complete(&adapter->rx_queue, adapter, budget);
1263 }
1264
1265
1266 static int
1267 vmxnet3_poll(struct napi_struct *napi, int budget)
1268 {
1269         struct vmxnet3_adapter *adapter = container_of(napi,
1270                                           struct vmxnet3_adapter, napi);
1271         int rxd_done;
1272
1273         rxd_done = vmxnet3_do_poll(adapter, budget);
1274
1275         if (rxd_done < budget) {
1276                 napi_complete(napi);
1277                 vmxnet3_enable_intr(adapter, 0);
1278         }
1279         return rxd_done;
1280 }
1281
1282
1283 /* Interrupt handler for vmxnet3  */
1284 static irqreturn_t
1285 vmxnet3_intr(int irq, void *dev_id)
1286 {
1287         struct net_device *dev = dev_id;
1288         struct vmxnet3_adapter *adapter = netdev_priv(dev);
1289
1290         if (unlikely(adapter->intr.type == VMXNET3_IT_INTX)) {
1291                 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1292                 if (unlikely(icr == 0))
1293                         /* not ours */
1294                         return IRQ_NONE;
1295         }
1296
1297
1298         /* disable intr if needed */
1299         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1300                 vmxnet3_disable_intr(adapter, 0);
1301
1302         napi_schedule(&adapter->napi);
1303
1304         return IRQ_HANDLED;
1305 }
1306
1307 #ifdef CONFIG_NET_POLL_CONTROLLER
1308
1309
1310 /* netpoll callback. */
1311 static void
1312 vmxnet3_netpoll(struct net_device *netdev)
1313 {
1314         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1315         int irq;
1316
1317         if (adapter->intr.type == VMXNET3_IT_MSIX)
1318                 irq = adapter->intr.msix_entries[0].vector;
1319         else
1320                 irq = adapter->pdev->irq;
1321
1322         disable_irq(irq);
1323         vmxnet3_intr(irq, netdev);
1324         enable_irq(irq);
1325 }
1326 #endif
1327
1328 static int
1329 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1330 {
1331         int err;
1332
1333         if (adapter->intr.type == VMXNET3_IT_MSIX) {
1334                 /* we only use 1 MSI-X vector */
1335                 err = request_irq(adapter->intr.msix_entries[0].vector,
1336                                   vmxnet3_intr, 0, adapter->netdev->name,
1337                                   adapter->netdev);
1338         } else if (adapter->intr.type == VMXNET3_IT_MSI) {
1339                 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1340                                   adapter->netdev->name, adapter->netdev);
1341         } else {
1342                 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1343                                   IRQF_SHARED, adapter->netdev->name,
1344                                   adapter->netdev);
1345         }
1346
1347         if (err)
1348                 printk(KERN_ERR "Failed to request irq %s (intr type:%d), error"
1349                        ":%d\n", adapter->netdev->name, adapter->intr.type, err);
1350
1351
1352         if (!err) {
1353                 int i;
1354                 /* init our intr settings */
1355                 for (i = 0; i < adapter->intr.num_intrs; i++)
1356                         adapter->intr.mod_levels[i] = UPT1_IML_ADAPTIVE;
1357
1358                 /* next setup intr index for all intr sources */
1359                 adapter->tx_queue.comp_ring.intr_idx = 0;
1360                 adapter->rx_queue.comp_ring.intr_idx = 0;
1361                 adapter->intr.event_intr_idx = 0;
1362
1363                 printk(KERN_INFO "%s: intr type %u, mode %u, %u vectors "
1364                        "allocated\n", adapter->netdev->name, adapter->intr.type,
1365                        adapter->intr.mask_mode, adapter->intr.num_intrs);
1366         }
1367
1368         return err;
1369 }
1370
1371
1372 static void
1373 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1374 {
1375         BUG_ON(adapter->intr.type == VMXNET3_IT_AUTO ||
1376                adapter->intr.num_intrs <= 0);
1377
1378         switch (adapter->intr.type) {
1379         case VMXNET3_IT_MSIX:
1380         {
1381                 int i;
1382
1383                 for (i = 0; i < adapter->intr.num_intrs; i++)
1384                         free_irq(adapter->intr.msix_entries[i].vector,
1385                                  adapter->netdev);
1386                 break;
1387         }
1388         case VMXNET3_IT_MSI:
1389                 free_irq(adapter->pdev->irq, adapter->netdev);
1390                 break;
1391         case VMXNET3_IT_INTX:
1392                 free_irq(adapter->pdev->irq, adapter->netdev);
1393                 break;
1394         default:
1395                 BUG_ON(true);
1396         }
1397 }
1398
1399
1400 static void
1401 vmxnet3_vlan_rx_register(struct net_device *netdev, struct vlan_group *grp)
1402 {
1403         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1404         struct Vmxnet3_DriverShared *shared = adapter->shared;
1405         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1406
1407         if (grp) {
1408                 /* add vlan rx stripping. */
1409                 if (adapter->netdev->features & NETIF_F_HW_VLAN_RX) {
1410                         int i;
1411                         struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
1412                         adapter->vlan_grp = grp;
1413
1414                         /* update FEATURES to device */
1415                         devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
1416                         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1417                                                VMXNET3_CMD_UPDATE_FEATURE);
1418                         /*
1419                          *  Clear entire vfTable; then enable untagged pkts.
1420                          *  Note: setting one entry in vfTable to non-zero turns
1421                          *  on VLAN rx filtering.
1422                          */
1423                         for (i = 0; i < VMXNET3_VFT_SIZE; i++)
1424                                 vfTable[i] = 0;
1425
1426                         VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1427                         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1428                                                VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1429                 } else {
1430                         printk(KERN_ERR "%s: vlan_rx_register when device has "
1431                                "no NETIF_F_HW_VLAN_RX\n", netdev->name);
1432                 }
1433         } else {
1434                 /* remove vlan rx stripping. */
1435                 struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
1436                 adapter->vlan_grp = NULL;
1437
1438                 if (devRead->misc.uptFeatures & UPT1_F_RXVLAN) {
1439                         int i;
1440
1441                         for (i = 0; i < VMXNET3_VFT_SIZE; i++) {
1442                                 /* clear entire vfTable; this also disables
1443                                  * VLAN rx filtering
1444                                  */
1445                                 vfTable[i] = 0;
1446                         }
1447                         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1448                                                VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1449
1450                         /* update FEATURES to device */
1451                         devRead->misc.uptFeatures &= ~UPT1_F_RXVLAN;
1452                         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1453                                                VMXNET3_CMD_UPDATE_FEATURE);
1454                 }
1455         }
1456 }
1457
1458
1459 static void
1460 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1461 {
1462         if (adapter->vlan_grp) {
1463                 u16 vid;
1464                 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1465                 bool activeVlan = false;
1466
1467                 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
1468                         if (vlan_group_get_device(adapter->vlan_grp, vid)) {
1469                                 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1470                                 activeVlan = true;
1471                         }
1472                 }
1473                 if (activeVlan) {
1474                         /* continue to allow untagged pkts */
1475                         VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1476                 }
1477         }
1478 }
1479
1480
1481 static void
1482 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
1483 {
1484         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1485         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1486
1487         VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1488         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1489                                VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1490 }
1491
1492
1493 static void
1494 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
1495 {
1496         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1497         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1498
1499         VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1500         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1501                                VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1502 }
1503
1504
1505 static u8 *
1506 vmxnet3_copy_mc(struct net_device *netdev)
1507 {
1508         u8 *buf = NULL;
1509         u32 sz = netdev->mc_count * ETH_ALEN;
1510
1511         /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
1512         if (sz <= 0xffff) {
1513                 /* We may be called with BH disabled */
1514                 buf = kmalloc(sz, GFP_ATOMIC);
1515                 if (buf) {
1516                         int i;
1517                         struct dev_mc_list *mc = netdev->mc_list;
1518
1519                         for (i = 0; i < netdev->mc_count; i++) {
1520                                 BUG_ON(!mc);
1521                                 memcpy(buf + i * ETH_ALEN, mc->dmi_addr,
1522                                        ETH_ALEN);
1523                                 mc = mc->next;
1524                         }
1525                 }
1526         }
1527         return buf;
1528 }
1529
1530
1531 static void
1532 vmxnet3_set_mc(struct net_device *netdev)
1533 {
1534         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1535         struct Vmxnet3_RxFilterConf *rxConf =
1536                                         &adapter->shared->devRead.rxFilterConf;
1537         u8 *new_table = NULL;
1538         u32 new_mode = VMXNET3_RXM_UCAST;
1539
1540         if (netdev->flags & IFF_PROMISC)
1541                 new_mode |= VMXNET3_RXM_PROMISC;
1542
1543         if (netdev->flags & IFF_BROADCAST)
1544                 new_mode |= VMXNET3_RXM_BCAST;
1545
1546         if (netdev->flags & IFF_ALLMULTI)
1547                 new_mode |= VMXNET3_RXM_ALL_MULTI;
1548         else
1549                 if (netdev->mc_count > 0) {
1550                         new_table = vmxnet3_copy_mc(netdev);
1551                         if (new_table) {
1552                                 new_mode |= VMXNET3_RXM_MCAST;
1553                                 rxConf->mfTableLen = netdev->mc_count *
1554                                                      ETH_ALEN;
1555                                 rxConf->mfTablePA = virt_to_phys(new_table);
1556                         } else {
1557                                 printk(KERN_INFO "%s: failed to copy mcast list"
1558                                        ", setting ALL_MULTI\n", netdev->name);
1559                                 new_mode |= VMXNET3_RXM_ALL_MULTI;
1560                         }
1561                 }
1562
1563
1564         if (!(new_mode & VMXNET3_RXM_MCAST)) {
1565                 rxConf->mfTableLen = 0;
1566                 rxConf->mfTablePA = 0;
1567         }
1568
1569         if (new_mode != rxConf->rxMode) {
1570                 rxConf->rxMode = new_mode;
1571                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1572                                        VMXNET3_CMD_UPDATE_RX_MODE);
1573         }
1574
1575         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1576                                VMXNET3_CMD_UPDATE_MAC_FILTERS);
1577
1578         kfree(new_table);
1579 }
1580
1581
1582 /*
1583  *   Set up driver_shared based on settings in adapter.
1584  */
1585
1586 static void
1587 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
1588 {
1589         struct Vmxnet3_DriverShared *shared = adapter->shared;
1590         struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
1591         struct Vmxnet3_TxQueueConf *tqc;
1592         struct Vmxnet3_RxQueueConf *rqc;
1593         int i;
1594
1595         memset(shared, 0, sizeof(*shared));
1596
1597         /* driver settings */
1598         shared->magic = VMXNET3_REV1_MAGIC;
1599         devRead->misc.driverInfo.version = VMXNET3_DRIVER_VERSION_NUM;
1600         devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
1601                                 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
1602         devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
1603         devRead->misc.driverInfo.vmxnet3RevSpt = 1;
1604         devRead->misc.driverInfo.uptVerSpt = 1;
1605
1606         devRead->misc.ddPA = virt_to_phys(adapter);
1607         devRead->misc.ddLen = sizeof(struct vmxnet3_adapter);
1608
1609         /* set up feature flags */
1610         if (adapter->rxcsum)
1611                 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
1612
1613         if (adapter->lro) {
1614                 devRead->misc.uptFeatures |= UPT1_F_LRO;
1615                 devRead->misc.maxNumRxSG = 1 + MAX_SKB_FRAGS;
1616         }
1617         if ((adapter->netdev->features & NETIF_F_HW_VLAN_RX)
1618                         && adapter->vlan_grp) {
1619                 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
1620         }
1621
1622         devRead->misc.mtu = adapter->netdev->mtu;
1623         devRead->misc.queueDescPA = adapter->queue_desc_pa;
1624         devRead->misc.queueDescLen = sizeof(struct Vmxnet3_TxQueueDesc) +
1625                                      sizeof(struct Vmxnet3_RxQueueDesc);
1626
1627         /* tx queue settings */
1628         BUG_ON(adapter->tx_queue.tx_ring.base == NULL);
1629
1630         devRead->misc.numTxQueues = 1;
1631         tqc = &adapter->tqd_start->conf;
1632         tqc->txRingBasePA   = adapter->tx_queue.tx_ring.basePA;
1633         tqc->dataRingBasePA = adapter->tx_queue.data_ring.basePA;
1634         tqc->compRingBasePA = adapter->tx_queue.comp_ring.basePA;
1635         tqc->ddPA           = virt_to_phys(adapter->tx_queue.buf_info);
1636         tqc->txRingSize     = adapter->tx_queue.tx_ring.size;
1637         tqc->dataRingSize   = adapter->tx_queue.data_ring.size;
1638         tqc->compRingSize   = adapter->tx_queue.comp_ring.size;
1639         tqc->ddLen          = sizeof(struct vmxnet3_tx_buf_info) *
1640                               tqc->txRingSize;
1641         tqc->intrIdx        = adapter->tx_queue.comp_ring.intr_idx;
1642
1643         /* rx queue settings */
1644         devRead->misc.numRxQueues = 1;
1645         rqc = &adapter->rqd_start->conf;
1646         rqc->rxRingBasePA[0] = adapter->rx_queue.rx_ring[0].basePA;
1647         rqc->rxRingBasePA[1] = adapter->rx_queue.rx_ring[1].basePA;
1648         rqc->compRingBasePA  = adapter->rx_queue.comp_ring.basePA;
1649         rqc->ddPA            = virt_to_phys(adapter->rx_queue.buf_info);
1650         rqc->rxRingSize[0]   = adapter->rx_queue.rx_ring[0].size;
1651         rqc->rxRingSize[1]   = adapter->rx_queue.rx_ring[1].size;
1652         rqc->compRingSize    = adapter->rx_queue.comp_ring.size;
1653         rqc->ddLen           = sizeof(struct vmxnet3_rx_buf_info) *
1654                                (rqc->rxRingSize[0] + rqc->rxRingSize[1]);
1655         rqc->intrIdx         = adapter->rx_queue.comp_ring.intr_idx;
1656
1657         /* intr settings */
1658         devRead->intrConf.autoMask = adapter->intr.mask_mode ==
1659                                      VMXNET3_IMM_AUTO;
1660         devRead->intrConf.numIntrs = adapter->intr.num_intrs;
1661         for (i = 0; i < adapter->intr.num_intrs; i++)
1662                 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
1663
1664         devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
1665
1666         /* rx filter settings */
1667         devRead->rxFilterConf.rxMode = 0;
1668         vmxnet3_restore_vlan(adapter);
1669         /* the rest are already zeroed */
1670 }
1671
1672
1673 int
1674 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
1675 {
1676         int err;
1677         u32 ret;
1678
1679         dprintk(KERN_ERR "%s: skb_buf_size %d, rx_buf_per_pkt %d, ring sizes"
1680                 " %u %u %u\n", adapter->netdev->name, adapter->skb_buf_size,
1681                 adapter->rx_buf_per_pkt, adapter->tx_queue.tx_ring.size,
1682                 adapter->rx_queue.rx_ring[0].size,
1683                 adapter->rx_queue.rx_ring[1].size);
1684
1685         vmxnet3_tq_init(&adapter->tx_queue, adapter);
1686         err = vmxnet3_rq_init(&adapter->rx_queue, adapter);
1687         if (err) {
1688                 printk(KERN_ERR "Failed to init rx queue for %s: error %d\n",
1689                        adapter->netdev->name, err);
1690                 goto rq_err;
1691         }
1692
1693         err = vmxnet3_request_irqs(adapter);
1694         if (err) {
1695                 printk(KERN_ERR "Failed to setup irq for %s: error %d\n",
1696                        adapter->netdev->name, err);
1697                 goto irq_err;
1698         }
1699
1700         vmxnet3_setup_driver_shared(adapter);
1701
1702         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL,
1703                                VMXNET3_GET_ADDR_LO(adapter->shared_pa));
1704         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH,
1705                                VMXNET3_GET_ADDR_HI(adapter->shared_pa));
1706
1707         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1708                                VMXNET3_CMD_ACTIVATE_DEV);
1709         ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
1710
1711         if (ret != 0) {
1712                 printk(KERN_ERR "Failed to activate dev %s: error %u\n",
1713                        adapter->netdev->name, ret);
1714                 err = -EINVAL;
1715                 goto activate_err;
1716         }
1717         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_RXPROD,
1718                                adapter->rx_queue.rx_ring[0].next2fill);
1719         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_RXPROD2,
1720                                adapter->rx_queue.rx_ring[1].next2fill);
1721
1722         /* Apply the rx filter settins last. */
1723         vmxnet3_set_mc(adapter->netdev);
1724
1725         /*
1726          * Check link state when first activating device. It will start the
1727          * tx queue if the link is up.
1728          */
1729         vmxnet3_check_link(adapter);
1730
1731         napi_enable(&adapter->napi);
1732         vmxnet3_enable_all_intrs(adapter);
1733         clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
1734         return 0;
1735
1736 activate_err:
1737         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
1738         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
1739         vmxnet3_free_irqs(adapter);
1740 irq_err:
1741 rq_err:
1742         /* free up buffers we allocated */
1743         vmxnet3_rq_cleanup(&adapter->rx_queue, adapter);
1744         return err;
1745 }
1746
1747
1748 void
1749 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
1750 {
1751         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
1752 }
1753
1754
1755 int
1756 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
1757 {
1758         if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
1759                 return 0;
1760
1761
1762         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1763                                VMXNET3_CMD_QUIESCE_DEV);
1764         vmxnet3_disable_all_intrs(adapter);
1765
1766         napi_disable(&adapter->napi);
1767         netif_tx_disable(adapter->netdev);
1768         adapter->link_speed = 0;
1769         netif_carrier_off(adapter->netdev);
1770
1771         vmxnet3_tq_cleanup(&adapter->tx_queue, adapter);
1772         vmxnet3_rq_cleanup(&adapter->rx_queue, adapter);
1773         vmxnet3_free_irqs(adapter);
1774         return 0;
1775 }
1776
1777
1778 static void
1779 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
1780 {
1781         u32 tmp;
1782
1783         tmp = *(u32 *)mac;
1784         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
1785
1786         tmp = (mac[5] << 8) | mac[4];
1787         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
1788 }
1789
1790
1791 static int
1792 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
1793 {
1794         struct sockaddr *addr = p;
1795         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1796
1797         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1798         vmxnet3_write_mac_addr(adapter, addr->sa_data);
1799
1800         return 0;
1801 }
1802
1803
1804 /* ==================== initialization and cleanup routines ============ */
1805
1806 static int
1807 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
1808 {
1809         int err;
1810         unsigned long mmio_start, mmio_len;
1811         struct pci_dev *pdev = adapter->pdev;
1812
1813         err = pci_enable_device(pdev);
1814         if (err) {
1815                 printk(KERN_ERR "Failed to enable adapter %s: error %d\n",
1816                        pci_name(pdev), err);
1817                 return err;
1818         }
1819
1820         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
1821                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
1822                         printk(KERN_ERR "pci_set_consistent_dma_mask failed "
1823                                "for adapter %s\n", pci_name(pdev));
1824                         err = -EIO;
1825                         goto err_set_mask;
1826                 }
1827                 *dma64 = true;
1828         } else {
1829                 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
1830                         printk(KERN_ERR "pci_set_dma_mask failed for adapter "
1831                                "%s\n",  pci_name(pdev));
1832                         err = -EIO;
1833                         goto err_set_mask;
1834                 }
1835                 *dma64 = false;
1836         }
1837
1838         err = pci_request_selected_regions(pdev, (1 << 2) - 1,
1839                                            vmxnet3_driver_name);
1840         if (err) {
1841                 printk(KERN_ERR "Failed to request region for adapter %s: "
1842                        "error %d\n", pci_name(pdev), err);
1843                 goto err_set_mask;
1844         }
1845
1846         pci_set_master(pdev);
1847
1848         mmio_start = pci_resource_start(pdev, 0);
1849         mmio_len = pci_resource_len(pdev, 0);
1850         adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
1851         if (!adapter->hw_addr0) {
1852                 printk(KERN_ERR "Failed to map bar0 for adapter %s\n",
1853                        pci_name(pdev));
1854                 err = -EIO;
1855                 goto err_ioremap;
1856         }
1857
1858         mmio_start = pci_resource_start(pdev, 1);
1859         mmio_len = pci_resource_len(pdev, 1);
1860         adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
1861         if (!adapter->hw_addr1) {
1862                 printk(KERN_ERR "Failed to map bar1 for adapter %s\n",
1863                        pci_name(pdev));
1864                 err = -EIO;
1865                 goto err_bar1;
1866         }
1867         return 0;
1868
1869 err_bar1:
1870         iounmap(adapter->hw_addr0);
1871 err_ioremap:
1872         pci_release_selected_regions(pdev, (1 << 2) - 1);
1873 err_set_mask:
1874         pci_disable_device(pdev);
1875         return err;
1876 }
1877
1878
1879 static void
1880 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
1881 {
1882         BUG_ON(!adapter->pdev);
1883
1884         iounmap(adapter->hw_addr0);
1885         iounmap(adapter->hw_addr1);
1886         pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
1887         pci_disable_device(adapter->pdev);
1888 }
1889
1890
1891 static void
1892 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
1893 {
1894         size_t sz;
1895
1896         if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
1897                                     VMXNET3_MAX_ETH_HDR_SIZE) {
1898                 adapter->skb_buf_size = adapter->netdev->mtu +
1899                                         VMXNET3_MAX_ETH_HDR_SIZE;
1900                 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
1901                         adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
1902
1903                 adapter->rx_buf_per_pkt = 1;
1904         } else {
1905                 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
1906                 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
1907                                             VMXNET3_MAX_ETH_HDR_SIZE;
1908                 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
1909         }
1910
1911         /*
1912          * for simplicity, force the ring0 size to be a multiple of
1913          * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
1914          */
1915         sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
1916         adapter->rx_queue.rx_ring[0].size = (adapter->rx_queue.rx_ring[0].size +
1917                                              sz - 1) / sz * sz;
1918         adapter->rx_queue.rx_ring[0].size = min_t(u32,
1919                                             adapter->rx_queue.rx_ring[0].size,
1920                                             VMXNET3_RX_RING_MAX_SIZE / sz * sz);
1921 }
1922
1923
1924 int
1925 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
1926                       u32 rx_ring_size, u32 rx_ring2_size)
1927 {
1928         int err;
1929
1930         adapter->tx_queue.tx_ring.size   = tx_ring_size;
1931         adapter->tx_queue.data_ring.size = tx_ring_size;
1932         adapter->tx_queue.comp_ring.size = tx_ring_size;
1933         adapter->tx_queue.shared = &adapter->tqd_start->ctrl;
1934         adapter->tx_queue.stopped = true;
1935         err = vmxnet3_tq_create(&adapter->tx_queue, adapter);
1936         if (err)
1937                 return err;
1938
1939         adapter->rx_queue.rx_ring[0].size = rx_ring_size;
1940         adapter->rx_queue.rx_ring[1].size = rx_ring2_size;
1941         vmxnet3_adjust_rx_ring_size(adapter);
1942         adapter->rx_queue.comp_ring.size  = adapter->rx_queue.rx_ring[0].size +
1943                                             adapter->rx_queue.rx_ring[1].size;
1944         adapter->rx_queue.qid  = 0;
1945         adapter->rx_queue.qid2 = 1;
1946         adapter->rx_queue.shared = &adapter->rqd_start->ctrl;
1947         err = vmxnet3_rq_create(&adapter->rx_queue, adapter);
1948         if (err)
1949                 vmxnet3_tq_destroy(&adapter->tx_queue, adapter);
1950
1951         return err;
1952 }
1953
1954 static int
1955 vmxnet3_open(struct net_device *netdev)
1956 {
1957         struct vmxnet3_adapter *adapter;
1958         int err;
1959
1960         adapter = netdev_priv(netdev);
1961
1962         spin_lock_init(&adapter->tx_queue.tx_lock);
1963
1964         err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
1965                                     VMXNET3_DEF_RX_RING_SIZE,
1966                                     VMXNET3_DEF_RX_RING_SIZE);
1967         if (err)
1968                 goto queue_err;
1969
1970         err = vmxnet3_activate_dev(adapter);
1971         if (err)
1972                 goto activate_err;
1973
1974         return 0;
1975
1976 activate_err:
1977         vmxnet3_rq_destroy(&adapter->rx_queue, adapter);
1978         vmxnet3_tq_destroy(&adapter->tx_queue, adapter);
1979 queue_err:
1980         return err;
1981 }
1982
1983
1984 static int
1985 vmxnet3_close(struct net_device *netdev)
1986 {
1987         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1988
1989         /*
1990          * Reset_work may be in the middle of resetting the device, wait for its
1991          * completion.
1992          */
1993         while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
1994                 msleep(1);
1995
1996         vmxnet3_quiesce_dev(adapter);
1997
1998         vmxnet3_rq_destroy(&adapter->rx_queue, adapter);
1999         vmxnet3_tq_destroy(&adapter->tx_queue, adapter);
2000
2001         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2002
2003
2004         return 0;
2005 }
2006
2007
2008 void
2009 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2010 {
2011         /*
2012          * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2013          * vmxnet3_close() will deadlock.
2014          */
2015         BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2016
2017         /* we need to enable NAPI, otherwise dev_close will deadlock */
2018         napi_enable(&adapter->napi);
2019         dev_close(adapter->netdev);
2020 }
2021
2022
2023 static int
2024 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2025 {
2026         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2027         int err = 0;
2028
2029         if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2030                 return -EINVAL;
2031
2032         if (new_mtu > 1500 && !adapter->jumbo_frame)
2033                 return -EINVAL;
2034
2035         netdev->mtu = new_mtu;
2036
2037         /*
2038          * Reset_work may be in the middle of resetting the device, wait for its
2039          * completion.
2040          */
2041         while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2042                 msleep(1);
2043
2044         if (netif_running(netdev)) {
2045                 vmxnet3_quiesce_dev(adapter);
2046                 vmxnet3_reset_dev(adapter);
2047
2048                 /* we need to re-create the rx queue based on the new mtu */
2049                 vmxnet3_rq_destroy(&adapter->rx_queue, adapter);
2050                 vmxnet3_adjust_rx_ring_size(adapter);
2051                 adapter->rx_queue.comp_ring.size  =
2052                                         adapter->rx_queue.rx_ring[0].size +
2053                                         adapter->rx_queue.rx_ring[1].size;
2054                 err = vmxnet3_rq_create(&adapter->rx_queue, adapter);
2055                 if (err) {
2056                         printk(KERN_ERR "%s: failed to re-create rx queue,"
2057                                 " error %d. Closing it.\n", netdev->name, err);
2058                         goto out;
2059                 }
2060
2061                 err = vmxnet3_activate_dev(adapter);
2062                 if (err) {
2063                         printk(KERN_ERR "%s: failed to re-activate, error %d. "
2064                                 "Closing it\n", netdev->name, err);
2065                         goto out;
2066                 }
2067         }
2068
2069 out:
2070         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2071         if (err)
2072                 vmxnet3_force_close(adapter);
2073
2074         return err;
2075 }
2076
2077
2078 static void
2079 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2080 {
2081         struct net_device *netdev = adapter->netdev;
2082
2083         netdev->features = NETIF_F_SG |
2084                 NETIF_F_HW_CSUM |
2085                 NETIF_F_HW_VLAN_TX |
2086                 NETIF_F_HW_VLAN_RX |
2087                 NETIF_F_HW_VLAN_FILTER |
2088                 NETIF_F_TSO |
2089                 NETIF_F_TSO6 |
2090                 NETIF_F_LRO;
2091
2092         printk(KERN_INFO "features: sg csum vlan jf tso tsoIPv6 lro");
2093
2094         adapter->rxcsum = true;
2095         adapter->jumbo_frame = true;
2096         adapter->lro = true;
2097
2098         if (dma64) {
2099                 netdev->features |= NETIF_F_HIGHDMA;
2100                 printk(" highDMA");
2101         }
2102
2103         netdev->vlan_features = netdev->features;
2104         printk("\n");
2105 }
2106
2107
2108 static void
2109 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2110 {
2111         u32 tmp;
2112
2113         tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2114         *(u32 *)mac = tmp;
2115
2116         tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2117         mac[4] = tmp & 0xff;
2118         mac[5] = (tmp >> 8) & 0xff;
2119 }
2120
2121
2122 static void
2123 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2124 {
2125         u32 cfg;
2126
2127         /* intr settings */
2128         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2129                                VMXNET3_CMD_GET_CONF_INTR);
2130         cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2131         adapter->intr.type = cfg & 0x3;
2132         adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2133
2134         if (adapter->intr.type == VMXNET3_IT_AUTO) {
2135                 int err;
2136
2137                 adapter->intr.msix_entries[0].entry = 0;
2138                 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
2139                                       VMXNET3_LINUX_MAX_MSIX_VECT);
2140                 if (!err) {
2141                         adapter->intr.num_intrs = 1;
2142                         adapter->intr.type = VMXNET3_IT_MSIX;
2143                         return;
2144                 }
2145
2146                 err = pci_enable_msi(adapter->pdev);
2147                 if (!err) {
2148                         adapter->intr.num_intrs = 1;
2149                         adapter->intr.type = VMXNET3_IT_MSI;
2150                         return;
2151                 }
2152         }
2153
2154         adapter->intr.type = VMXNET3_IT_INTX;
2155
2156         /* INT-X related setting */
2157         adapter->intr.num_intrs = 1;
2158 }
2159
2160
2161 static void
2162 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2163 {
2164         if (adapter->intr.type == VMXNET3_IT_MSIX)
2165                 pci_disable_msix(adapter->pdev);
2166         else if (adapter->intr.type == VMXNET3_IT_MSI)
2167                 pci_disable_msi(adapter->pdev);
2168         else
2169                 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2170 }
2171
2172
2173 static void
2174 vmxnet3_tx_timeout(struct net_device *netdev)
2175 {
2176         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2177         adapter->tx_timeout_count++;
2178
2179         printk(KERN_ERR "%s: tx hang\n", adapter->netdev->name);
2180         schedule_work(&adapter->work);
2181 }
2182
2183
2184 static void
2185 vmxnet3_reset_work(struct work_struct *data)
2186 {
2187         struct vmxnet3_adapter *adapter;
2188
2189         adapter = container_of(data, struct vmxnet3_adapter, work);
2190
2191         /* if another thread is resetting the device, no need to proceed */
2192         if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2193                 return;
2194
2195         /* if the device is closed, we must leave it alone */
2196         if (netif_running(adapter->netdev)) {
2197                 printk(KERN_INFO "%s: resetting\n", adapter->netdev->name);
2198                 vmxnet3_quiesce_dev(adapter);
2199                 vmxnet3_reset_dev(adapter);
2200                 vmxnet3_activate_dev(adapter);
2201         } else {
2202                 printk(KERN_INFO "%s: already closed\n", adapter->netdev->name);
2203         }
2204
2205         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2206 }
2207
2208
2209 static int __devinit
2210 vmxnet3_probe_device(struct pci_dev *pdev,
2211                      const struct pci_device_id *id)
2212 {
2213         static const struct net_device_ops vmxnet3_netdev_ops = {
2214                 .ndo_open = vmxnet3_open,
2215                 .ndo_stop = vmxnet3_close,
2216                 .ndo_start_xmit = vmxnet3_xmit_frame,
2217                 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2218                 .ndo_change_mtu = vmxnet3_change_mtu,
2219                 .ndo_get_stats = vmxnet3_get_stats,
2220                 .ndo_tx_timeout = vmxnet3_tx_timeout,
2221                 .ndo_set_multicast_list = vmxnet3_set_mc,
2222                 .ndo_vlan_rx_register = vmxnet3_vlan_rx_register,
2223                 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2224                 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2225 #ifdef CONFIG_NET_POLL_CONTROLLER
2226                 .ndo_poll_controller = vmxnet3_netpoll,
2227 #endif
2228         };
2229         int err;
2230         bool dma64 = false; /* stupid gcc */
2231         u32 ver;
2232         struct net_device *netdev;
2233         struct vmxnet3_adapter *adapter;
2234         u8 mac[ETH_ALEN];
2235
2236         netdev = alloc_etherdev(sizeof(struct vmxnet3_adapter));
2237         if (!netdev) {
2238                 printk(KERN_ERR "Failed to alloc ethernet device for adapter "
2239                         "%s\n", pci_name(pdev));
2240                 return -ENOMEM;
2241         }
2242
2243         pci_set_drvdata(pdev, netdev);
2244         adapter = netdev_priv(netdev);
2245         adapter->netdev = netdev;
2246         adapter->pdev = pdev;
2247
2248         adapter->shared = pci_alloc_consistent(adapter->pdev,
2249                           sizeof(struct Vmxnet3_DriverShared),
2250                           &adapter->shared_pa);
2251         if (!adapter->shared) {
2252                 printk(KERN_ERR "Failed to allocate memory for %s\n",
2253                         pci_name(pdev));
2254                 err = -ENOMEM;
2255                 goto err_alloc_shared;
2256         }
2257
2258         adapter->tqd_start = pci_alloc_consistent(adapter->pdev,
2259                              sizeof(struct Vmxnet3_TxQueueDesc) +
2260                              sizeof(struct Vmxnet3_RxQueueDesc),
2261                              &adapter->queue_desc_pa);
2262
2263         if (!adapter->tqd_start) {
2264                 printk(KERN_ERR "Failed to allocate memory for %s\n",
2265                         pci_name(pdev));
2266                 err = -ENOMEM;
2267                 goto err_alloc_queue_desc;
2268         }
2269         adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start
2270                                                             + 1);
2271
2272         adapter->pm_conf = kmalloc(sizeof(struct Vmxnet3_PMConf), GFP_KERNEL);
2273         if (adapter->pm_conf == NULL) {
2274                 printk(KERN_ERR "Failed to allocate memory for %s\n",
2275                         pci_name(pdev));
2276                 err = -ENOMEM;
2277                 goto err_alloc_pm;
2278         }
2279
2280         err = vmxnet3_alloc_pci_resources(adapter, &dma64);
2281         if (err < 0)
2282                 goto err_alloc_pci;
2283
2284         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
2285         if (ver & 1) {
2286                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
2287         } else {
2288                 printk(KERN_ERR "Incompatible h/w version (0x%x) for adapter"
2289                        " %s\n", ver, pci_name(pdev));
2290                 err = -EBUSY;
2291                 goto err_ver;
2292         }
2293
2294         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
2295         if (ver & 1) {
2296                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
2297         } else {
2298                 printk(KERN_ERR "Incompatible upt version (0x%x) for "
2299                        "adapter %s\n", ver, pci_name(pdev));
2300                 err = -EBUSY;
2301                 goto err_ver;
2302         }
2303
2304         vmxnet3_declare_features(adapter, dma64);
2305
2306         adapter->dev_number = atomic_read(&devices_found);
2307         vmxnet3_alloc_intr_resources(adapter);
2308
2309         vmxnet3_read_mac_addr(adapter, mac);
2310         memcpy(netdev->dev_addr,  mac, netdev->addr_len);
2311
2312         netdev->netdev_ops = &vmxnet3_netdev_ops;
2313         netdev->watchdog_timeo = 5 * HZ;
2314         vmxnet3_set_ethtool_ops(netdev);
2315
2316         INIT_WORK(&adapter->work, vmxnet3_reset_work);
2317
2318         netif_napi_add(netdev, &adapter->napi, vmxnet3_poll, 64);
2319         SET_NETDEV_DEV(netdev, &pdev->dev);
2320         err = register_netdev(netdev);
2321
2322         if (err) {
2323                 printk(KERN_ERR "Failed to register adapter %s\n",
2324                         pci_name(pdev));
2325                 goto err_register;
2326         }
2327
2328         set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2329         atomic_inc(&devices_found);
2330         return 0;
2331
2332 err_register:
2333         vmxnet3_free_intr_resources(adapter);
2334 err_ver:
2335         vmxnet3_free_pci_resources(adapter);
2336 err_alloc_pci:
2337         kfree(adapter->pm_conf);
2338 err_alloc_pm:
2339         pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_TxQueueDesc) +
2340                             sizeof(struct Vmxnet3_RxQueueDesc),
2341                             adapter->tqd_start, adapter->queue_desc_pa);
2342 err_alloc_queue_desc:
2343         pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
2344                             adapter->shared, adapter->shared_pa);
2345 err_alloc_shared:
2346         pci_set_drvdata(pdev, NULL);
2347         free_netdev(netdev);
2348         return err;
2349 }
2350
2351
2352 static void __devexit
2353 vmxnet3_remove_device(struct pci_dev *pdev)
2354 {
2355         struct net_device *netdev = pci_get_drvdata(pdev);
2356         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2357
2358         flush_scheduled_work();
2359
2360         unregister_netdev(netdev);
2361
2362         vmxnet3_free_intr_resources(adapter);
2363         vmxnet3_free_pci_resources(adapter);
2364         kfree(adapter->pm_conf);
2365         pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_TxQueueDesc) +
2366                             sizeof(struct Vmxnet3_RxQueueDesc),
2367                             adapter->tqd_start, adapter->queue_desc_pa);
2368         pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
2369                             adapter->shared, adapter->shared_pa);
2370         free_netdev(netdev);
2371 }
2372
2373
2374 #ifdef CONFIG_PM
2375
2376 static int
2377 vmxnet3_suspend(struct device *device)
2378 {
2379         struct pci_dev *pdev = to_pci_dev(device);
2380         struct net_device *netdev = pci_get_drvdata(pdev);
2381         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2382         struct Vmxnet3_PMConf *pmConf;
2383         struct ethhdr *ehdr;
2384         struct arphdr *ahdr;
2385         u8 *arpreq;
2386         struct in_device *in_dev;
2387         struct in_ifaddr *ifa;
2388         int i = 0;
2389
2390         if (!netif_running(netdev))
2391                 return 0;
2392
2393         vmxnet3_disable_all_intrs(adapter);
2394         vmxnet3_free_irqs(adapter);
2395         vmxnet3_free_intr_resources(adapter);
2396
2397         netif_device_detach(netdev);
2398         netif_stop_queue(netdev);
2399
2400         /* Create wake-up filters. */
2401         pmConf = adapter->pm_conf;
2402         memset(pmConf, 0, sizeof(*pmConf));
2403
2404         if (adapter->wol & WAKE_UCAST) {
2405                 pmConf->filters[i].patternSize = ETH_ALEN;
2406                 pmConf->filters[i].maskSize = 1;
2407                 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
2408                 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
2409
2410                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
2411                 i++;
2412         }
2413
2414         if (adapter->wol & WAKE_ARP) {
2415                 in_dev = in_dev_get(netdev);
2416                 if (!in_dev)
2417                         goto skip_arp;
2418
2419                 ifa = (struct in_ifaddr *)in_dev->ifa_list;
2420                 if (!ifa)
2421                         goto skip_arp;
2422
2423                 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
2424                         sizeof(struct arphdr) +         /* ARP header */
2425                         2 * ETH_ALEN +          /* 2 Ethernet addresses*/
2426                         2 * sizeof(u32);        /*2 IPv4 addresses */
2427                 pmConf->filters[i].maskSize =
2428                         (pmConf->filters[i].patternSize - 1) / 8 + 1;
2429
2430                 /* ETH_P_ARP in Ethernet header. */
2431                 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
2432                 ehdr->h_proto = htons(ETH_P_ARP);
2433
2434                 /* ARPOP_REQUEST in ARP header. */
2435                 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
2436                 ahdr->ar_op = htons(ARPOP_REQUEST);
2437                 arpreq = (u8 *)(ahdr + 1);
2438
2439                 /* The Unicast IPv4 address in 'tip' field. */
2440                 arpreq += 2 * ETH_ALEN + sizeof(u32);
2441                 *(u32 *)arpreq = ifa->ifa_address;
2442
2443                 /* The mask for the relevant bits. */
2444                 pmConf->filters[i].mask[0] = 0x00;
2445                 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
2446                 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
2447                 pmConf->filters[i].mask[3] = 0x00;
2448                 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
2449                 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
2450                 in_dev_put(in_dev);
2451
2452                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
2453                 i++;
2454         }
2455
2456 skip_arp:
2457         if (adapter->wol & WAKE_MAGIC)
2458                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
2459
2460         pmConf->numFilters = i;
2461
2462         adapter->shared->devRead.pmConfDesc.confVer = 1;
2463         adapter->shared->devRead.pmConfDesc.confLen = sizeof(*pmConf);
2464         adapter->shared->devRead.pmConfDesc.confPA = virt_to_phys(pmConf);
2465
2466         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2467                                VMXNET3_CMD_UPDATE_PMCFG);
2468
2469         pci_save_state(pdev);
2470         pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
2471                         adapter->wol);
2472         pci_disable_device(pdev);
2473         pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
2474
2475         return 0;
2476 }
2477
2478
2479 static int
2480 vmxnet3_resume(struct device *device)
2481 {
2482         int err;
2483         struct pci_dev *pdev = to_pci_dev(device);
2484         struct net_device *netdev = pci_get_drvdata(pdev);
2485         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2486         struct Vmxnet3_PMConf *pmConf;
2487
2488         if (!netif_running(netdev))
2489                 return 0;
2490
2491         /* Destroy wake-up filters. */
2492         pmConf = adapter->pm_conf;
2493         memset(pmConf, 0, sizeof(*pmConf));
2494
2495         adapter->shared->devRead.pmConfDesc.confVer = 1;
2496         adapter->shared->devRead.pmConfDesc.confLen = sizeof(*pmConf);
2497         adapter->shared->devRead.pmConfDesc.confPA = virt_to_phys(pmConf);
2498
2499         netif_device_attach(netdev);
2500         pci_set_power_state(pdev, PCI_D0);
2501         pci_restore_state(pdev);
2502         err = pci_enable_device_mem(pdev);
2503         if (err != 0)
2504                 return err;
2505
2506         pci_enable_wake(pdev, PCI_D0, 0);
2507
2508         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2509                                VMXNET3_CMD_UPDATE_PMCFG);
2510         vmxnet3_alloc_intr_resources(adapter);
2511         vmxnet3_request_irqs(adapter);
2512         vmxnet3_enable_all_intrs(adapter);
2513
2514         return 0;
2515 }
2516
2517 static struct dev_pm_ops vmxnet3_pm_ops = {
2518         .suspend = vmxnet3_suspend,
2519         .resume = vmxnet3_resume,
2520 };
2521 #endif
2522
2523 static struct pci_driver vmxnet3_driver = {
2524         .name           = vmxnet3_driver_name,
2525         .id_table       = vmxnet3_pciid_table,
2526         .probe          = vmxnet3_probe_device,
2527         .remove         = __devexit_p(vmxnet3_remove_device),
2528 #ifdef CONFIG_PM
2529         .driver.pm      = &vmxnet3_pm_ops,
2530 #endif
2531 };
2532
2533
2534 static int __init
2535 vmxnet3_init_module(void)
2536 {
2537         printk(KERN_INFO "%s - version %s\n", VMXNET3_DRIVER_DESC,
2538                 VMXNET3_DRIVER_VERSION_REPORT);
2539         return pci_register_driver(&vmxnet3_driver);
2540 }
2541
2542 module_init(vmxnet3_init_module);
2543
2544
2545 static void
2546 vmxnet3_exit_module(void)
2547 {
2548         pci_unregister_driver(&vmxnet3_driver);
2549 }
2550
2551 module_exit(vmxnet3_exit_module);
2552
2553 MODULE_AUTHOR("VMware, Inc.");
2554 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
2555 MODULE_LICENSE("GPL v2");
2556 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);