SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/net/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2009 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/module.h>
  29 #include <linux/types.h>
  30 #include <linux/init.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/ipv6.h>
  35 #include <net/checksum.h>
  36 #include <net/ip6_checksum.h>
  37 #include <linux/net_tstamp.h>
  38 #include <linux/mii.h>
  39 #include <linux/ethtool.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/pci.h>
  42 #include <linux/pci-aspm.h>
  43 #include <linux/delay.h>
  44 #include <linux/interrupt.h>
  45 #include <linux/if_ether.h>
  46 #include <linux/aer.h>
  47 #ifdef CONFIG_IGB_DCA
  48 #include <linux/dca.h>
  49 #endif
  50 #include "igb.h"
  51
  52 #define DRV_VERSION "2.1.0-k2"
  53 char igb_driver_name[] = "igb";
  54 char igb_driver_version[] = DRV_VERSION;
  55 static const char igb_driver_string[] =
  56                                 "Intel(R) Gigabit Ethernet Network Driver";
  57 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
  58
  59 static const struct e1000_info *igb_info_tbl[] = {
  60         [board_82575] = &e1000_82575_info,
  61 };
  62
  63 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  64         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  65         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  66         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  67         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  68         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  69         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  70         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
  71         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
  72         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
  73         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
  74         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
  75         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
  76         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
  79         /* required last entry */
  80         {0, }
  81 };
  82
  83 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
  84
  85 void igb_reset(struct igb_adapter *);
  86 static int igb_setup_all_tx_resources(struct igb_adapter *);
  87 static int igb_setup_all_rx_resources(struct igb_adapter *);
  88 static void igb_free_all_tx_resources(struct igb_adapter *);
  89 static void igb_free_all_rx_resources(struct igb_adapter *);
  90 static void igb_setup_mrqc(struct igb_adapter *);
  91 void igb_update_stats(struct igb_adapter *);
  92 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
  93 static void __devexit igb_remove(struct pci_dev *pdev);
  94 static int igb_sw_init(struct igb_adapter *);
  95 static int igb_open(struct net_device *);
  96 static int igb_close(struct net_device *);
  97 static void igb_configure_tx(struct igb_adapter *);
  98 static void igb_configure_rx(struct igb_adapter *);
  99 static void igb_clean_all_tx_rings(struct igb_adapter *);
 100 static void igb_clean_all_rx_rings(struct igb_adapter *);
 101 static void igb_clean_tx_ring(struct igb_ring *);
 102 static void igb_clean_rx_ring(struct igb_ring *);
 103 static void igb_set_rx_mode(struct net_device *);
 104 static void igb_update_phy_info(unsigned long);
 105 static void igb_watchdog(unsigned long);
 106 static void igb_watchdog_task(struct work_struct *);
 107 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
 108 static struct net_device_stats *igb_get_stats(struct net_device *);
 109 static int igb_change_mtu(struct net_device *, int);
 110 static int igb_set_mac(struct net_device *, void *);
 111 static void igb_set_uta(struct igb_adapter *adapter);
 112 static irqreturn_t igb_intr(int irq, void *);
 113 static irqreturn_t igb_intr_msi(int irq, void *);
 114 static irqreturn_t igb_msix_other(int irq, void *);
 115 static irqreturn_t igb_msix_ring(int irq, void *);
 116 #ifdef CONFIG_IGB_DCA
 117 static void igb_update_dca(struct igb_q_vector *);
 118 static void igb_setup_dca(struct igb_adapter *);
 119 #endif /* CONFIG_IGB_DCA */
 120 static bool igb_clean_tx_irq(struct igb_q_vector *);
 121 static int igb_poll(struct napi_struct *, int);
 122 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
 123 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 124 static void igb_tx_timeout(struct net_device *);
 125 static void igb_reset_task(struct work_struct *);
 126 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
 127 static void igb_vlan_rx_add_vid(struct net_device *, u16);
 128 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
 129 static void igb_restore_vlan(struct igb_adapter *);
 130 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
 131 static void igb_ping_all_vfs(struct igb_adapter *);
 132 static void igb_msg_task(struct igb_adapter *);
 133 static void igb_vmm_control(struct igb_adapter *);
 134 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 135 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 136
 137 #ifdef CONFIG_PM
 138 static int igb_suspend(struct pci_dev *, pm_message_t);
 139 static int igb_resume(struct pci_dev *);
 140 #endif
 141 static void igb_shutdown(struct pci_dev *);
 142 #ifdef CONFIG_IGB_DCA
 143 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 144 static struct notifier_block dca_notifier = {
 145         .notifier_call  = igb_notify_dca,
 146         .next           = NULL,
 147         .priority       = 0
 148 };
 149 #endif
 150 #ifdef CONFIG_NET_POLL_CONTROLLER
 151 /* for netdump / net console */
 152 static void igb_netpoll(struct net_device *);
 153 #endif
 154 #ifdef CONFIG_PCI_IOV
 155 static unsigned int max_vfs = 0;
 156 module_param(max_vfs, uint, 0);
 157 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
 158                  "per physical function");
 159 #endif /* CONFIG_PCI_IOV */
 160
 161 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 162                      pci_channel_state_t);
 163 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 164 static void igb_io_resume(struct pci_dev *);
 165
 166 static struct pci_error_handlers igb_err_handler = {
 167         .error_detected = igb_io_error_detected,
 168         .slot_reset = igb_io_slot_reset,
 169         .resume = igb_io_resume,
 170 };
 171
 172
 173 static struct pci_driver igb_driver = {
 174         .name     = igb_driver_name,
 175         .id_table = igb_pci_tbl,
 176         .probe    = igb_probe,
 177         .remove   = __devexit_p(igb_remove),
 178 #ifdef CONFIG_PM
 179         /* Power Managment Hooks */
 180         .suspend  = igb_suspend,
 181         .resume   = igb_resume,
 182 #endif
 183         .shutdown = igb_shutdown,
 184         .err_handler = &igb_err_handler
 185 };
 186
 187 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 188 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 189 MODULE_LICENSE("GPL");
 190 MODULE_VERSION(DRV_VERSION);
 191
 192 /**
 193  * igb_read_clock - read raw cycle counter (to be used by time counter)
 194  */
 195 static cycle_t igb_read_clock(const struct cyclecounter *tc)
 196 {
 197         struct igb_adapter *adapter =
 198                 container_of(tc, struct igb_adapter, cycles);
 199         struct e1000_hw *hw = &adapter->hw;
 200         u64 stamp = 0;
 201         int shift = 0;
 202
 203         /*
 204          * The timestamp latches on lowest register read. For the 82580
 205          * the lowest register is SYSTIMR instead of SYSTIML.  However we never
 206          * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
 207          */
 208         if (hw->mac.type == e1000_82580) {
 209                 stamp = rd32(E1000_SYSTIMR) >> 8;
 210                 shift = IGB_82580_TSYNC_SHIFT;
 211         }
 212
 213         stamp |= (u64)rd32(E1000_SYSTIML) << shift;
 214         stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
 215         return stamp;
 216 }
 217
 218 #ifdef DEBUG
 219 /**
 220  * igb_get_hw_dev_name - return device name string
 221  * used by hardware layer to print debugging information
 222  **/
 223 char *igb_get_hw_dev_name(struct e1000_hw *hw)
 224 {
 225         struct igb_adapter *adapter = hw->back;
 226         return adapter->netdev->name;
 227 }
 228
 229 /**
 230  * igb_get_time_str - format current NIC and system time as string
 231  */
 232 static char *igb_get_time_str(struct igb_adapter *adapter,
 233                               char buffer[160])
 234 {
 235         cycle_t hw = adapter->cycles.read(&adapter->cycles);
 236         struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
 237         struct timespec sys;
 238         struct timespec delta;
 239         getnstimeofday(&sys);
 240
 241         delta = timespec_sub(nic, sys);
 242
 243         sprintf(buffer,
 244                 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
 245                 hw,
 246                 (long)nic.tv_sec, nic.tv_nsec,
 247                 (long)sys.tv_sec, sys.tv_nsec,
 248                 (long)delta.tv_sec, delta.tv_nsec);
 249
 250         return buffer;
 251 }
 252 #endif
 253
 254 /**
 255  * igb_init_module - Driver Registration Routine
 256  *
 257  * igb_init_module is the first routine called when the driver is
 258  * loaded. All it does is register with the PCI subsystem.
 259  **/
 260 static int __init igb_init_module(void)
 261 {
 262         int ret;
 263         printk(KERN_INFO "%s - version %s\n",
 264                igb_driver_string, igb_driver_version);
 265
 266         printk(KERN_INFO "%s\n", igb_copyright);
 267
 268 #ifdef CONFIG_IGB_DCA
 269         dca_register_notify(&dca_notifier);
 270 #endif
 271         ret = pci_register_driver(&igb_driver);
 272         return ret;
 273 }
 274
 275 module_init(igb_init_module);
 276
 277 /**
 278  * igb_exit_module - Driver Exit Cleanup Routine
 279  *
 280  * igb_exit_module is called just before the driver is removed
 281  * from memory.
 282  **/
 283 static void __exit igb_exit_module(void)
 284 {
 285 #ifdef CONFIG_IGB_DCA
 286         dca_unregister_notify(&dca_notifier);
 287 #endif
 288         pci_unregister_driver(&igb_driver);
 289 }
 290
 291 module_exit(igb_exit_module);
 292
 293 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 294 /**
 295  * igb_cache_ring_register - Descriptor ring to register mapping
 296  * @adapter: board private structure to initialize
 297  *
 298  * Once we know the feature-set enabled for the device, we'll cache
 299  * the register offset the descriptor ring is assigned to.
 300  **/
 301 static void igb_cache_ring_register(struct igb_adapter *adapter)
 302 {
 303         int i = 0, j = 0;
 304         u32 rbase_offset = adapter->vfs_allocated_count;
 305
 306         switch (adapter->hw.mac.type) {
 307         case e1000_82576:
 308                 /* The queues are allocated for virtualization such that VF 0
 309                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 310                  * In order to avoid collision we start at the first free queue
 311                  * and continue consuming queues in the same sequence
 312                  */
 313                 if (adapter->vfs_allocated_count) {
 314                         for (; i < adapter->rss_queues; i++)
 315                                 adapter->rx_ring[i].reg_idx = rbase_offset +
 316                                                               Q_IDX_82576(i);
 317                         for (; j < adapter->rss_queues; j++)
 318                                 adapter->tx_ring[j].reg_idx = rbase_offset +
 319                                                               Q_IDX_82576(j);
 320                 }
 321         case e1000_82575:
 322         case e1000_82580:
 323         default:
 324                 for (; i < adapter->num_rx_queues; i++)
 325                         adapter->rx_ring[i].reg_idx = rbase_offset + i;
 326                 for (; j < adapter->num_tx_queues; j++)
 327                         adapter->tx_ring[j].reg_idx = rbase_offset + j;
 328                 break;
 329         }
 330 }
 331
 332 static void igb_free_queues(struct igb_adapter *adapter)
 333 {
 334         kfree(adapter->tx_ring);
 335         kfree(adapter->rx_ring);
 336
 337         adapter->tx_ring = NULL;
 338         adapter->rx_ring = NULL;
 339
 340         adapter->num_rx_queues = 0;
 341         adapter->num_tx_queues = 0;
 342 }
 343
 344 /**
 345  * igb_alloc_queues - Allocate memory for all rings
 346  * @adapter: board private structure to initialize
 347  *
 348  * We allocate one ring per queue at run-time since we don't know the
 349  * number of queues at compile-time.
 350  **/
 351 static int igb_alloc_queues(struct igb_adapter *adapter)
 352 {
 353         int i;
 354
 355         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
 356                                    sizeof(struct igb_ring), GFP_KERNEL);
 357         if (!adapter->tx_ring)
 358                 goto err;
 359
 360         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
 361                                    sizeof(struct igb_ring), GFP_KERNEL);
 362         if (!adapter->rx_ring)
 363                 goto err;
 364
 365         for (i = 0; i < adapter->num_tx_queues; i++) {
 366                 struct igb_ring *ring = &(adapter->tx_ring[i]);
 367                 ring->count = adapter->tx_ring_count;
 368                 ring->queue_index = i;
 369                 ring->pdev = adapter->pdev;
 370                 ring->netdev = adapter->netdev;
 371                 /* For 82575, context index must be unique per ring. */
 372                 if (adapter->hw.mac.type == e1000_82575)
 373                         ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
 374         }
 375
 376         for (i = 0; i < adapter->num_rx_queues; i++) {
 377                 struct igb_ring *ring = &(adapter->rx_ring[i]);
 378                 ring->count = adapter->rx_ring_count;
 379                 ring->queue_index = i;
 380                 ring->pdev = adapter->pdev;
 381                 ring->netdev = adapter->netdev;
 382                 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
 383                 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
 384                 /* set flag indicating ring supports SCTP checksum offload */
 385                 if (adapter->hw.mac.type >= e1000_82576)
 386                         ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
 387         }
 388
 389         igb_cache_ring_register(adapter);
 390
 391         return 0;
 392
 393 err:
 394         igb_free_queues(adapter);
 395
 396         return -ENOMEM;
 397 }
 398
 399 #define IGB_N0_QUEUE -1
 400 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 401 {
 402         u32 msixbm = 0;
 403         struct igb_adapter *adapter = q_vector->adapter;
 404         struct e1000_hw *hw = &adapter->hw;
 405         u32 ivar, index;
 406         int rx_queue = IGB_N0_QUEUE;
 407         int tx_queue = IGB_N0_QUEUE;
 408
 409         if (q_vector->rx_ring)
 410                 rx_queue = q_vector->rx_ring->reg_idx;
 411         if (q_vector->tx_ring)
 412                 tx_queue = q_vector->tx_ring->reg_idx;
 413
 414         switch (hw->mac.type) {
 415         case e1000_82575:
 416                 /* The 82575 assigns vectors using a bitmask, which matches the
 417                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 418                    or more queues to a vector, we write the appropriate bits
 419                    into the MSIXBM register for that vector. */
 420                 if (rx_queue > IGB_N0_QUEUE)
 421                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 422                 if (tx_queue > IGB_N0_QUEUE)
 423                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 424                 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
 425                 q_vector->eims_value = msixbm;
 426                 break;
 427         case e1000_82576:
 428                 /* 82576 uses a table-based method for assigning vectors.
 429                    Each queue has a single entry in the table to which we write
 430                    a vector number along with a "valid" bit.  Sadly, the layout
 431                    of the table is somewhat counterintuitive. */
 432                 if (rx_queue > IGB_N0_QUEUE) {
 433                         index = (rx_queue & 0x7);
 434                         ivar = array_rd32(E1000_IVAR0, index);
 435                         if (rx_queue < 8) {
 436                                 /* vector goes into low byte of register */
 437                                 ivar = ivar & 0xFFFFFF00;
 438                                 ivar |= msix_vector | E1000_IVAR_VALID;
 439                         } else {
 440                                 /* vector goes into third byte of register */
 441                                 ivar = ivar & 0xFF00FFFF;
 442                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 443                         }
 444                         array_wr32(E1000_IVAR0, index, ivar);
 445                 }
 446                 if (tx_queue > IGB_N0_QUEUE) {
 447                         index = (tx_queue & 0x7);
 448                         ivar = array_rd32(E1000_IVAR0, index);
 449                         if (tx_queue < 8) {
 450                                 /* vector goes into second byte of register */
 451                                 ivar = ivar & 0xFFFF00FF;
 452                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 453                         } else {
 454                                 /* vector goes into high byte of register */
 455                                 ivar = ivar & 0x00FFFFFF;
 456                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 457                         }
 458                         array_wr32(E1000_IVAR0, index, ivar);
 459                 }
 460                 q_vector->eims_value = 1 << msix_vector;
 461                 break;
 462         case e1000_82580:
 463                 /* 82580 uses the same table-based approach as 82576 but has fewer
 464                    entries as a result we carry over for queues greater than 4. */
 465                 if (rx_queue > IGB_N0_QUEUE) {
 466                         index = (rx_queue >> 1);
 467                         ivar = array_rd32(E1000_IVAR0, index);
 468                         if (rx_queue & 0x1) {
 469                                 /* vector goes into third byte of register */
 470                                 ivar = ivar & 0xFF00FFFF;
 471                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 472                         } else {
 473                                 /* vector goes into low byte of register */
 474                                 ivar = ivar & 0xFFFFFF00;
 475                                 ivar |= msix_vector | E1000_IVAR_VALID;
 476                         }
 477                         array_wr32(E1000_IVAR0, index, ivar);
 478                 }
 479                 if (tx_queue > IGB_N0_QUEUE) {
 480                         index = (tx_queue >> 1);
 481                         ivar = array_rd32(E1000_IVAR0, index);
 482                         if (tx_queue & 0x1) {
 483                                 /* vector goes into high byte of register */
 484                                 ivar = ivar & 0x00FFFFFF;
 485                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 486                         } else {
 487                                 /* vector goes into second byte of register */
 488                                 ivar = ivar & 0xFFFF00FF;
 489                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 490                         }
 491                         array_wr32(E1000_IVAR0, index, ivar);
 492                 }
 493                 q_vector->eims_value = 1 << msix_vector;
 494                 break;
 495         default:
 496                 BUG();
 497                 break;
 498         }
 499 }
 500
 501 /**
 502  * igb_configure_msix - Configure MSI-X hardware
 503  *
 504  * igb_configure_msix sets up the hardware to properly
 505  * generate MSI-X interrupts.
 506  **/
 507 static void igb_configure_msix(struct igb_adapter *adapter)
 508 {
 509         u32 tmp;
 510         int i, vector = 0;
 511         struct e1000_hw *hw = &adapter->hw;
 512
 513         adapter->eims_enable_mask = 0;
 514
 515         /* set vector for other causes, i.e. link changes */
 516         switch (hw->mac.type) {
 517         case e1000_82575:
 518                 tmp = rd32(E1000_CTRL_EXT);
 519                 /* enable MSI-X PBA support*/
 520                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 521
 522                 /* Auto-Mask interrupts upon ICR read. */
 523                 tmp |= E1000_CTRL_EXT_EIAME;
 524                 tmp |= E1000_CTRL_EXT_IRCA;
 525
 526                 wr32(E1000_CTRL_EXT, tmp);
 527
 528                 /* enable msix_other interrupt */
 529                 array_wr32(E1000_MSIXBM(0), vector++,
 530                                       E1000_EIMS_OTHER);
 531                 adapter->eims_other = E1000_EIMS_OTHER;
 532
 533                 break;
 534
 535         case e1000_82576:
 536         case e1000_82580:
 537                 /* Turn on MSI-X capability first, or our settings
 538                  * won't stick.  And it will take days to debug. */
 539                 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
 540                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 541                                 E1000_GPIE_NSICR);
 542
 543                 /* enable msix_other interrupt */
 544                 adapter->eims_other = 1 << vector;
 545                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 546
 547                 wr32(E1000_IVAR_MISC, tmp);
 548                 break;
 549         default:
 550                 /* do nothing, since nothing else supports MSI-X */
 551                 break;
 552         } /* switch (hw->mac.type) */
 553
 554         adapter->eims_enable_mask |= adapter->eims_other;
 555
 556         for (i = 0; i < adapter->num_q_vectors; i++) {
 557                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 558                 igb_assign_vector(q_vector, vector++);
 559                 adapter->eims_enable_mask |= q_vector->eims_value;
 560         }
 561
 562         wrfl();
 563 }
 564
 565 /**
 566  * igb_request_msix - Initialize MSI-X interrupts
 567  *
 568  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 569  * kernel.
 570  **/
 571 static int igb_request_msix(struct igb_adapter *adapter)
 572 {
 573         struct net_device *netdev = adapter->netdev;
 574         struct e1000_hw *hw = &adapter->hw;
 575         int i, err = 0, vector = 0;
 576
 577         err = request_irq(adapter->msix_entries[vector].vector,
 578                           igb_msix_other, 0, netdev->name, adapter);
 579         if (err)
 580                 goto out;
 581         vector++;
 582
 583         for (i = 0; i < adapter->num_q_vectors; i++) {
 584                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 585
 586                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 587
 588                 if (q_vector->rx_ring && q_vector->tx_ring)
 589                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 590                                 q_vector->rx_ring->queue_index);
 591                 else if (q_vector->tx_ring)
 592                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 593                                 q_vector->tx_ring->queue_index);
 594                 else if (q_vector->rx_ring)
 595                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 596                                 q_vector->rx_ring->queue_index);
 597                 else
 598                         sprintf(q_vector->name, "%s-unused", netdev->name);
 599
 600                 err = request_irq(adapter->msix_entries[vector].vector,
 601                                   igb_msix_ring, 0, q_vector->name,
 602                                   q_vector);
 603                 if (err)
 604                         goto out;
 605                 vector++;
 606         }
 607
 608         igb_configure_msix(adapter);
 609         return 0;
 610 out:
 611         return err;
 612 }
 613
 614 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 615 {
 616         if (adapter->msix_entries) {
 617                 pci_disable_msix(adapter->pdev);
 618                 kfree(adapter->msix_entries);
 619                 adapter->msix_entries = NULL;
 620         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 621                 pci_disable_msi(adapter->pdev);
 622         }
 623 }
 624
 625 /**
 626  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 627  * @adapter: board private structure to initialize
 628  *
 629  * This function frees the memory allocated to the q_vectors.  In addition if
 630  * NAPI is enabled it will delete any references to the NAPI struct prior
 631  * to freeing the q_vector.
 632  **/
 633 static void igb_free_q_vectors(struct igb_adapter *adapter)
 634 {
 635         int v_idx;
 636
 637         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 638                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 639                 adapter->q_vector[v_idx] = NULL;
 640                 netif_napi_del(&q_vector->napi);
 641                 kfree(q_vector);
 642         }
 643         adapter->num_q_vectors = 0;
 644 }
 645
 646 /**
 647  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 648  *
 649  * This function resets the device so that it has 0 rx queues, tx queues, and
 650  * MSI-X interrupts allocated.
 651  */
 652 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 653 {
 654         igb_free_queues(adapter);
 655         igb_free_q_vectors(adapter);
 656         igb_reset_interrupt_capability(adapter);
 657 }
 658
 659 /**
 660  * igb_set_interrupt_capability - set MSI or MSI-X if supported
 661  *
 662  * Attempt to configure interrupts using the best available
 663  * capabilities of the hardware and kernel.
 664  **/
 665 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
 666 {
 667         int err;
 668         int numvecs, i;
 669
 670         /* Number of supported queues. */
 671         adapter->num_rx_queues = adapter->rss_queues;
 672         adapter->num_tx_queues = adapter->rss_queues;
 673
 674         /* start with one vector for every rx queue */
 675         numvecs = adapter->num_rx_queues;
 676
 677         /* if tx handler is seperate add 1 for every tx queue */
 678         if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
 679                 numvecs += adapter->num_tx_queues;
 680
 681         /* store the number of vectors reserved for queues */
 682         adapter->num_q_vectors = numvecs;
 683
 684         /* add 1 vector for link status interrupts */
 685         numvecs++;
 686         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
 687                                         GFP_KERNEL);
 688         if (!adapter->msix_entries)
 689                 goto msi_only;
 690
 691         for (i = 0; i < numvecs; i++)
 692                 adapter->msix_entries[i].entry = i;
 693
 694         err = pci_enable_msix(adapter->pdev,
 695                               adapter->msix_entries,
 696                               numvecs);
 697         if (err == 0)
 698                 goto out;
 699
 700         igb_reset_interrupt_capability(adapter);
 701
 702         /* If we can't do MSI-X, try MSI */
 703 msi_only:
 704 #ifdef CONFIG_PCI_IOV
 705         /* disable SR-IOV for non MSI-X configurations */
 706         if (adapter->vf_data) {
 707                 struct e1000_hw *hw = &adapter->hw;
 708                 /* disable iov and allow time for transactions to clear */
 709                 pci_disable_sriov(adapter->pdev);
 710                 msleep(500);
 711
 712                 kfree(adapter->vf_data);
 713                 adapter->vf_data = NULL;
 714                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
 715                 msleep(100);
 716                 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
 717         }
 718 #endif
 719         adapter->vfs_allocated_count = 0;
 720         adapter->rss_queues = 1;
 721         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
 722         adapter->num_rx_queues = 1;
 723         adapter->num_tx_queues = 1;
 724         adapter->num_q_vectors = 1;
 725         if (!pci_enable_msi(adapter->pdev))
 726                 adapter->flags |= IGB_FLAG_HAS_MSI;
 727 out:
 728         /* Notify the stack of the (possibly) reduced Tx Queue count. */
 729         adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
 730         return;
 731 }
 732
 733 /**
 734  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
 735  * @adapter: board private structure to initialize
 736  *
 737  * We allocate one q_vector per queue interrupt.  If allocation fails we
 738  * return -ENOMEM.
 739  **/
 740 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
 741 {
 742         struct igb_q_vector *q_vector;
 743         struct e1000_hw *hw = &adapter->hw;
 744         int v_idx;
 745
 746         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 747                 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
 748                 if (!q_vector)
 749                         goto err_out;
 750                 q_vector->adapter = adapter;
 751                 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
 752                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
 753                 q_vector->itr_val = IGB_START_ITR;
 754                 q_vector->set_itr = 1;
 755                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
 756                 adapter->q_vector[v_idx] = q_vector;
 757         }
 758         return 0;
 759
 760 err_out:
 761         while (v_idx) {
 762                 v_idx--;
 763                 q_vector = adapter->q_vector[v_idx];
 764                 netif_napi_del(&q_vector->napi);
 765                 kfree(q_vector);
 766                 adapter->q_vector[v_idx] = NULL;
 767         }
 768         return -ENOMEM;
 769 }
 770
 771 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
 772                                       int ring_idx, int v_idx)
 773 {
 774         struct igb_q_vector *q_vector;
 775
 776         q_vector = adapter->q_vector[v_idx];
 777         q_vector->rx_ring = &adapter->rx_ring[ring_idx];
 778         q_vector->rx_ring->q_vector = q_vector;
 779         q_vector->itr_val = adapter->rx_itr_setting;
 780         if (q_vector->itr_val && q_vector->itr_val <= 3)
 781                 q_vector->itr_val = IGB_START_ITR;
 782 }
 783
 784 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
 785                                       int ring_idx, int v_idx)
 786 {
 787         struct igb_q_vector *q_vector;
 788
 789         q_vector = adapter->q_vector[v_idx];
 790         q_vector->tx_ring = &adapter->tx_ring[ring_idx];
 791         q_vector->tx_ring->q_vector = q_vector;
 792         q_vector->itr_val = adapter->tx_itr_setting;
 793         if (q_vector->itr_val && q_vector->itr_val <= 3)
 794                 q_vector->itr_val = IGB_START_ITR;
 795 }
 796
 797 /**
 798  * igb_map_ring_to_vector - maps allocated queues to vectors
 799  *
 800  * This function maps the recently allocated queues to vectors.
 801  **/
 802 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
 803 {
 804         int i;
 805         int v_idx = 0;
 806
 807         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
 808             (adapter->num_q_vectors < adapter->num_tx_queues))
 809                 return -ENOMEM;
 810
 811         if (adapter->num_q_vectors >=
 812             (adapter->num_rx_queues + adapter->num_tx_queues)) {
 813                 for (i = 0; i < adapter->num_rx_queues; i++)
 814                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
 815                 for (i = 0; i < adapter->num_tx_queues; i++)
 816                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
 817         } else {
 818                 for (i = 0; i < adapter->num_rx_queues; i++) {
 819                         if (i < adapter->num_tx_queues)
 820                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
 821                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
 822                 }
 823                 for (; i < adapter->num_tx_queues; i++)
 824                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
 825         }
 826         return 0;
 827 }
 828
 829 /**
 830  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
 831  *
 832  * This function initializes the interrupts and allocates all of the queues.
 833  **/
 834 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
 835 {
 836         struct pci_dev *pdev = adapter->pdev;
 837         int err;
 838
 839         igb_set_interrupt_capability(adapter);
 840
 841         err = igb_alloc_q_vectors(adapter);
 842         if (err) {
 843                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
 844                 goto err_alloc_q_vectors;
 845         }
 846
 847         err = igb_alloc_queues(adapter);
 848         if (err) {
 849                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
 850                 goto err_alloc_queues;
 851         }
 852
 853         err = igb_map_ring_to_vector(adapter);
 854         if (err) {
 855                 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
 856                 goto err_map_queues;
 857         }
 858
 859
 860         return 0;
 861 err_map_queues:
 862         igb_free_queues(adapter);
 863 err_alloc_queues:
 864         igb_free_q_vectors(adapter);
 865 err_alloc_q_vectors:
 866         igb_reset_interrupt_capability(adapter);
 867         return err;
 868 }
 869
 870 /**
 871  * igb_request_irq - initialize interrupts
 872  *
 873  * Attempts to configure interrupts using the best available
 874  * capabilities of the hardware and kernel.
 875  **/
 876 static int igb_request_irq(struct igb_adapter *adapter)
 877 {
 878         struct net_device *netdev = adapter->netdev;
 879         struct pci_dev *pdev = adapter->pdev;
 880         struct e1000_hw *hw = &adapter->hw;
 881         int err = 0;
 882
 883         if (adapter->msix_entries) {
 884                 err = igb_request_msix(adapter);
 885                 if (!err)
 886                         goto request_done;
 887                 /* fall back to MSI */
 888                 igb_clear_interrupt_scheme(adapter);
 889                 if (!pci_enable_msi(adapter->pdev))
 890                         adapter->flags |= IGB_FLAG_HAS_MSI;
 891                 igb_free_all_tx_resources(adapter);
 892                 igb_free_all_rx_resources(adapter);
 893                 adapter->num_tx_queues = 1;
 894                 adapter->num_rx_queues = 1;
 895                 adapter->num_q_vectors = 1;
 896                 err = igb_alloc_q_vectors(adapter);
 897                 if (err) {
 898                         dev_err(&pdev->dev,
 899                                 "Unable to allocate memory for vectors\n");
 900                         goto request_done;
 901                 }
 902                 err = igb_alloc_queues(adapter);
 903                 if (err) {
 904                         dev_err(&pdev->dev,
 905                                 "Unable to allocate memory for queues\n");
 906                         igb_free_q_vectors(adapter);
 907                         goto request_done;
 908                 }
 909                 igb_setup_all_tx_resources(adapter);
 910                 igb_setup_all_rx_resources(adapter);
 911         } else {
 912                 switch (hw->mac.type) {
 913                 case e1000_82575:
 914                         wr32(E1000_MSIXBM(0),
 915                              (E1000_EICR_RX_QUEUE0 |
 916                               E1000_EICR_TX_QUEUE0 |
 917                               E1000_EIMS_OTHER));
 918                         break;
 919                 case e1000_82580:
 920                 case e1000_82576:
 921                         wr32(E1000_IVAR0, E1000_IVAR_VALID);
 922                         break;
 923                 default:
 924                         break;
 925                 }
 926         }
 927
 928         if (adapter->flags & IGB_FLAG_HAS_MSI) {
 929                 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
 930                                   netdev->name, adapter);
 931                 if (!err)
 932                         goto request_done;
 933
 934                 /* fall back to legacy interrupts */
 935                 igb_reset_interrupt_capability(adapter);
 936                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
 937         }
 938
 939         err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
 940                           netdev->name, adapter);
 941
 942         if (err)
 943                 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
 944                         err);
 945
 946 request_done:
 947         return err;
 948 }
 949
 950 static void igb_free_irq(struct igb_adapter *adapter)
 951 {
 952         if (adapter->msix_entries) {
 953                 int vector = 0, i;
 954
 955                 free_irq(adapter->msix_entries[vector++].vector, adapter);
 956
 957                 for (i = 0; i < adapter->num_q_vectors; i++) {
 958                         struct igb_q_vector *q_vector = adapter->q_vector[i];
 959                         free_irq(adapter->msix_entries[vector++].vector,
 960                                  q_vector);
 961                 }
 962         } else {
 963                 free_irq(adapter->pdev->irq, adapter);
 964         }
 965 }
 966
 967 /**
 968  * igb_irq_disable - Mask off interrupt generation on the NIC
 969  * @adapter: board private structure
 970  **/
 971 static void igb_irq_disable(struct igb_adapter *adapter)
 972 {
 973         struct e1000_hw *hw = &adapter->hw;
 974
 975         /*
 976          * we need to be careful when disabling interrupts.  The VFs are also
 977          * mapped into these registers and so clearing the bits can cause
 978          * issues on the VF drivers so we only need to clear what we set
 979          */
 980         if (adapter->msix_entries) {
 981                 u32 regval = rd32(E1000_EIAM);
 982                 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
 983                 wr32(E1000_EIMC, adapter->eims_enable_mask);
 984                 regval = rd32(E1000_EIAC);
 985                 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
 986         }
 987
 988         wr32(E1000_IAM, 0);
 989         wr32(E1000_IMC, ~0);
 990         wrfl();
 991         synchronize_irq(adapter->pdev->irq);
 992 }
 993
 994 /**
 995  * igb_irq_enable - Enable default interrupt generation settings
 996  * @adapter: board private structure
 997  **/
 998 static void igb_irq_enable(struct igb_adapter *adapter)
 999 {
1000         struct e1000_hw *hw = &adapter->hw;
1001
1002         if (adapter->msix_entries) {
1003                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1004                 u32 regval = rd32(E1000_EIAC);
1005                 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1006                 regval = rd32(E1000_EIAM);
1007                 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1008                 wr32(E1000_EIMS, adapter->eims_enable_mask);
1009                 if (adapter->vfs_allocated_count) {
1010                         wr32(E1000_MBVFIMR, 0xFF);
1011                         ims |= E1000_IMS_VMMB;
1012                 }
1013                 if (adapter->hw.mac.type == e1000_82580)
1014                         ims |= E1000_IMS_DRSTA;
1015
1016                 wr32(E1000_IMS, ims);
1017         } else {
1018                 wr32(E1000_IMS, IMS_ENABLE_MASK |
1019                                 E1000_IMS_DRSTA);
1020                 wr32(E1000_IAM, IMS_ENABLE_MASK |
1021                                 E1000_IMS_DRSTA);
1022         }
1023 }
1024
1025 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1026 {
1027         struct e1000_hw *hw = &adapter->hw;
1028         u16 vid = adapter->hw.mng_cookie.vlan_id;
1029         u16 old_vid = adapter->mng_vlan_id;
1030
1031         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1032                 /* add VID to filter table */
1033                 igb_vfta_set(hw, vid, true);
1034                 adapter->mng_vlan_id = vid;
1035         } else {
1036                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1037         }
1038
1039         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1040             (vid != old_vid) &&
1041             !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1042                 /* remove VID from filter table */
1043                 igb_vfta_set(hw, old_vid, false);
1044         }
1045 }
1046
1047 /**
1048  * igb_release_hw_control - release control of the h/w to f/w
1049  * @adapter: address of board private structure
1050  *
1051  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1052  * For ASF and Pass Through versions of f/w this means that the
1053  * driver is no longer loaded.
1054  *
1055  **/
1056 static void igb_release_hw_control(struct igb_adapter *adapter)
1057 {
1058         struct e1000_hw *hw = &adapter->hw;
1059         u32 ctrl_ext;
1060
1061         /* Let firmware take over control of h/w */
1062         ctrl_ext = rd32(E1000_CTRL_EXT);
1063         wr32(E1000_CTRL_EXT,
1064                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1065 }
1066
1067 /**
1068  * igb_get_hw_control - get control of the h/w from f/w
1069  * @adapter: address of board private structure
1070  *
1071  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1072  * For ASF and Pass Through versions of f/w this means that
1073  * the driver is loaded.
1074  *
1075  **/
1076 static void igb_get_hw_control(struct igb_adapter *adapter)
1077 {
1078         struct e1000_hw *hw = &adapter->hw;
1079         u32 ctrl_ext;
1080
1081         /* Let firmware know the driver has taken over */
1082         ctrl_ext = rd32(E1000_CTRL_EXT);
1083         wr32(E1000_CTRL_EXT,
1084                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1085 }
1086
1087 /**
1088  * igb_configure - configure the hardware for RX and TX
1089  * @adapter: private board structure
1090  **/
1091 static void igb_configure(struct igb_adapter *adapter)
1092 {
1093         struct net_device *netdev = adapter->netdev;
1094         int i;
1095
1096         igb_get_hw_control(adapter);
1097         igb_set_rx_mode(netdev);
1098
1099         igb_restore_vlan(adapter);
1100
1101         igb_setup_tctl(adapter);
1102         igb_setup_mrqc(adapter);
1103         igb_setup_rctl(adapter);
1104
1105         igb_configure_tx(adapter);
1106         igb_configure_rx(adapter);
1107
1108         igb_rx_fifo_flush_82575(&adapter->hw);
1109
1110         /* call igb_desc_unused which always leaves
1111          * at least 1 descriptor unused to make sure
1112          * next_to_use != next_to_clean */
1113         for (i = 0; i < adapter->num_rx_queues; i++) {
1114                 struct igb_ring *ring = &adapter->rx_ring[i];
1115                 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1116         }
1117
1118
1119         adapter->tx_queue_len = netdev->tx_queue_len;
1120 }
1121
1122
1123 /**
1124  * igb_up - Open the interface and prepare it to handle traffic
1125  * @adapter: board private structure
1126  **/
1127 int igb_up(struct igb_adapter *adapter)
1128 {
1129         struct e1000_hw *hw = &adapter->hw;
1130         int i;
1131
1132         /* hardware has been reset, we need to reload some things */
1133         igb_configure(adapter);
1134
1135         clear_bit(__IGB_DOWN, &adapter->state);
1136
1137         for (i = 0; i < adapter->num_q_vectors; i++) {
1138                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1139                 napi_enable(&q_vector->napi);
1140         }
1141         if (adapter->msix_entries)
1142                 igb_configure_msix(adapter);
1143
1144         /* Clear any pending interrupts. */
1145         rd32(E1000_ICR);
1146         igb_irq_enable(adapter);
1147
1148         /* notify VFs that reset has been completed */
1149         if (adapter->vfs_allocated_count) {
1150                 u32 reg_data = rd32(E1000_CTRL_EXT);
1151                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1152                 wr32(E1000_CTRL_EXT, reg_data);
1153         }
1154
1155         netif_tx_start_all_queues(adapter->netdev);
1156
1157         /* start the watchdog. */
1158         hw->mac.get_link_status = 1;
1159         schedule_work(&adapter->watchdog_task);
1160
1161         return 0;
1162 }
1163
1164 void igb_down(struct igb_adapter *adapter)
1165 {
1166         struct net_device *netdev = adapter->netdev;
1167         struct e1000_hw *hw = &adapter->hw;
1168         u32 tctl, rctl;
1169         int i;
1170
1171         /* signal that we're down so the interrupt handler does not
1172          * reschedule our watchdog timer */
1173         set_bit(__IGB_DOWN, &adapter->state);
1174
1175         /* disable receives in the hardware */
1176         rctl = rd32(E1000_RCTL);
1177         wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1178         /* flush and sleep below */
1179
1180         netif_tx_stop_all_queues(netdev);
1181
1182         /* disable transmits in the hardware */
1183         tctl = rd32(E1000_TCTL);
1184         tctl &= ~E1000_TCTL_EN;
1185         wr32(E1000_TCTL, tctl);
1186         /* flush both disables and wait for them to finish */
1187         wrfl();
1188         msleep(10);
1189
1190         for (i = 0; i < adapter->num_q_vectors; i++) {
1191                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1192                 napi_disable(&q_vector->napi);
1193         }
1194
1195         igb_irq_disable(adapter);
1196
1197         del_timer_sync(&adapter->watchdog_timer);
1198         del_timer_sync(&adapter->phy_info_timer);
1199
1200         netdev->tx_queue_len = adapter->tx_queue_len;
1201         netif_carrier_off(netdev);
1202
1203         /* record the stats before reset*/
1204         igb_update_stats(adapter);
1205
1206         adapter->link_speed = 0;
1207         adapter->link_duplex = 0;
1208
1209         if (!pci_channel_offline(adapter->pdev))
1210                 igb_reset(adapter);
1211         igb_clean_all_tx_rings(adapter);
1212         igb_clean_all_rx_rings(adapter);
1213 #ifdef CONFIG_IGB_DCA
1214
1215         /* since we reset the hardware DCA settings were cleared */
1216         igb_setup_dca(adapter);
1217 #endif
1218 }
1219
1220 void igb_reinit_locked(struct igb_adapter *adapter)
1221 {
1222         WARN_ON(in_interrupt());
1223         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1224                 msleep(1);
1225         igb_down(adapter);
1226         igb_up(adapter);
1227         clear_bit(__IGB_RESETTING, &adapter->state);
1228 }
1229
1230 void igb_reset(struct igb_adapter *adapter)
1231 {
1232         struct pci_dev *pdev = adapter->pdev;
1233         struct e1000_hw *hw = &adapter->hw;
1234         struct e1000_mac_info *mac = &hw->mac;
1235         struct e1000_fc_info *fc = &hw->fc;
1236         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1237         u16 hwm;
1238
1239         /* Repartition Pba for greater than 9k mtu
1240          * To take effect CTRL.RST is required.
1241          */
1242         switch (mac->type) {
1243         case e1000_82580:
1244                 pba = rd32(E1000_RXPBS);
1245                 pba = igb_rxpbs_adjust_82580(pba);
1246                 break;
1247         case e1000_82576:
1248                 pba = rd32(E1000_RXPBS);
1249                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1250                 break;
1251         case e1000_82575:
1252         default:
1253                 pba = E1000_PBA_34K;
1254                 break;
1255         }
1256
1257         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1258             (mac->type < e1000_82576)) {
1259                 /* adjust PBA for jumbo frames */
1260                 wr32(E1000_PBA, pba);
1261
1262                 /* To maintain wire speed transmits, the Tx FIFO should be
1263                  * large enough to accommodate two full transmit packets,
1264                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1265                  * the Rx FIFO should be large enough to accommodate at least
1266                  * one full receive packet and is similarly rounded up and
1267                  * expressed in KB. */
1268                 pba = rd32(E1000_PBA);
1269                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1270                 tx_space = pba >> 16;
1271                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1272                 pba &= 0xffff;
1273                 /* the tx fifo also stores 16 bytes of information about the tx
1274                  * but don't include ethernet FCS because hardware appends it */
1275                 min_tx_space = (adapter->max_frame_size +
1276                                 sizeof(union e1000_adv_tx_desc) -
1277                                 ETH_FCS_LEN) * 2;
1278                 min_tx_space = ALIGN(min_tx_space, 1024);
1279                 min_tx_space >>= 10;
1280                 /* software strips receive CRC, so leave room for it */
1281                 min_rx_space = adapter->max_frame_size;
1282                 min_rx_space = ALIGN(min_rx_space, 1024);
1283                 min_rx_space >>= 10;
1284
1285                 /* If current Tx allocation is less than the min Tx FIFO size,
1286                  * and the min Tx FIFO size is less than the current Rx FIFO
1287                  * allocation, take space away from current Rx allocation */
1288                 if (tx_space < min_tx_space &&
1289                     ((min_tx_space - tx_space) < pba)) {
1290                         pba = pba - (min_tx_space - tx_space);
1291
1292                         /* if short on rx space, rx wins and must trump tx
1293                          * adjustment */
1294                         if (pba < min_rx_space)
1295                                 pba = min_rx_space;
1296                 }
1297                 wr32(E1000_PBA, pba);
1298         }
1299
1300         /* flow control settings */
1301         /* The high water mark must be low enough to fit one full frame
1302          * (or the size used for early receive) above it in the Rx FIFO.
1303          * Set it to the lower of:
1304          * - 90% of the Rx FIFO size, or
1305          * - the full Rx FIFO size minus one full frame */
1306         hwm = min(((pba << 10) * 9 / 10),
1307                         ((pba << 10) - 2 * adapter->max_frame_size));
1308
1309         fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1310         fc->low_water = fc->high_water - 16;
1311         fc->pause_time = 0xFFFF;
1312         fc->send_xon = 1;
1313         fc->current_mode = fc->requested_mode;
1314
1315         /* disable receive for all VFs and wait one second */
1316         if (adapter->vfs_allocated_count) {
1317                 int i;
1318                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1319                         adapter->vf_data[i].flags = 0;
1320
1321                 /* ping all the active vfs to let them know we are going down */
1322                 igb_ping_all_vfs(adapter);
1323
1324                 /* disable transmits and receives */
1325                 wr32(E1000_VFRE, 0);
1326                 wr32(E1000_VFTE, 0);
1327         }
1328
1329         /* Allow time for pending master requests to run */
1330         hw->mac.ops.reset_hw(hw);
1331         wr32(E1000_WUC, 0);
1332
1333         if (hw->mac.ops.init_hw(hw))
1334                 dev_err(&pdev->dev, "Hardware Error\n");
1335
1336         if (hw->mac.type == e1000_82580) {
1337                 u32 reg = rd32(E1000_PCIEMISC);
1338                 wr32(E1000_PCIEMISC,
1339                                 reg & ~E1000_PCIEMISC_LX_DECISION);
1340         }
1341         igb_update_mng_vlan(adapter);
1342
1343         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1344         wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1345
1346         igb_reset_adaptive(hw);
1347         igb_get_phy_info(hw);
1348 }
1349
1350 static const struct net_device_ops igb_netdev_ops = {
1351         .ndo_open               = igb_open,
1352         .ndo_stop               = igb_close,
1353         .ndo_start_xmit         = igb_xmit_frame_adv,
1354         .ndo_get_stats          = igb_get_stats,
1355         .ndo_set_rx_mode        = igb_set_rx_mode,
1356         .ndo_set_multicast_list = igb_set_rx_mode,
1357         .ndo_set_mac_address    = igb_set_mac,
1358         .ndo_change_mtu         = igb_change_mtu,
1359         .ndo_do_ioctl           = igb_ioctl,
1360         .ndo_tx_timeout         = igb_tx_timeout,
1361         .ndo_validate_addr      = eth_validate_addr,
1362         .ndo_vlan_rx_register   = igb_vlan_rx_register,
1363         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1364         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1365 #ifdef CONFIG_NET_POLL_CONTROLLER
1366         .ndo_poll_controller    = igb_netpoll,
1367 #endif
1368 };
1369
1370 /**
1371  * igb_probe - Device Initialization Routine
1372  * @pdev: PCI device information struct
1373  * @ent: entry in igb_pci_tbl
1374  *
1375  * Returns 0 on success, negative on failure
1376  *
1377  * igb_probe initializes an adapter identified by a pci_dev structure.
1378  * The OS initialization, configuring of the adapter private structure,
1379  * and a hardware reset occur.
1380  **/
1381 static int __devinit igb_probe(struct pci_dev *pdev,
1382                                const struct pci_device_id *ent)
1383 {
1384         struct net_device *netdev;
1385         struct igb_adapter *adapter;
1386         struct e1000_hw *hw;
1387         u16 eeprom_data = 0;
1388         static int global_quad_port_a; /* global quad port a indication */
1389         const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1390         unsigned long mmio_start, mmio_len;
1391         int err, pci_using_dac;
1392         u16 eeprom_apme_mask = IGB_EEPROM_APME;
1393         u32 part_num;
1394
1395         err = pci_enable_device_mem(pdev);
1396         if (err)
1397                 return err;
1398
1399         pci_using_dac = 0;
1400         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1401         if (!err) {
1402                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1403                 if (!err)
1404                         pci_using_dac = 1;
1405         } else {
1406                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1407                 if (err) {
1408                         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1409                         if (err) {
1410                                 dev_err(&pdev->dev, "No usable DMA "
1411                                         "configuration, aborting\n");
1412                                 goto err_dma;
1413                         }
1414                 }
1415         }
1416
1417         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1418                                            IORESOURCE_MEM),
1419                                            igb_driver_name);
1420         if (err)
1421                 goto err_pci_reg;
1422
1423         pci_enable_pcie_error_reporting(pdev);
1424
1425         pci_set_master(pdev);
1426         pci_save_state(pdev);
1427
1428         err = -ENOMEM;
1429         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1430                                    IGB_ABS_MAX_TX_QUEUES);
1431         if (!netdev)
1432                 goto err_alloc_etherdev;
1433
1434         SET_NETDEV_DEV(netdev, &pdev->dev);
1435
1436         pci_set_drvdata(pdev, netdev);
1437         adapter = netdev_priv(netdev);
1438         adapter->netdev = netdev;
1439         adapter->pdev = pdev;
1440         hw = &adapter->hw;
1441         hw->back = adapter;
1442         adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1443
1444         mmio_start = pci_resource_start(pdev, 0);
1445         mmio_len = pci_resource_len(pdev, 0);
1446
1447         err = -EIO;
1448         hw->hw_addr = ioremap(mmio_start, mmio_len);
1449         if (!hw->hw_addr)
1450                 goto err_ioremap;
1451
1452         netdev->netdev_ops = &igb_netdev_ops;
1453         igb_set_ethtool_ops(netdev);
1454         netdev->watchdog_timeo = 5 * HZ;
1455
1456         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1457
1458         netdev->mem_start = mmio_start;
1459         netdev->mem_end = mmio_start + mmio_len;
1460
1461         /* PCI config space info */
1462         hw->vendor_id = pdev->vendor;
1463         hw->device_id = pdev->device;
1464         hw->revision_id = pdev->revision;
1465         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1466         hw->subsystem_device_id = pdev->subsystem_device;
1467
1468         /* Copy the default MAC, PHY and NVM function pointers */
1469         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1470         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1471         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1472         /* Initialize skew-specific constants */
1473         err = ei->get_invariants(hw);
1474         if (err)
1475                 goto err_sw_init;
1476
1477         /* setup the private structure */
1478         err = igb_sw_init(adapter);
1479         if (err)
1480                 goto err_sw_init;
1481
1482         igb_get_bus_info_pcie(hw);
1483
1484         hw->phy.autoneg_wait_to_complete = false;
1485         hw->mac.adaptive_ifs = true;
1486
1487         /* Copper options */
1488         if (hw->phy.media_type == e1000_media_type_copper) {
1489                 hw->phy.mdix = AUTO_ALL_MODES;
1490                 hw->phy.disable_polarity_correction = false;
1491                 hw->phy.ms_type = e1000_ms_hw_default;
1492         }
1493
1494         if (igb_check_reset_block(hw))
1495                 dev_info(&pdev->dev,
1496                         "PHY reset is blocked due to SOL/IDER session.\n");
1497
1498         netdev->features = NETIF_F_SG |
1499                            NETIF_F_IP_CSUM |
1500                            NETIF_F_HW_VLAN_TX |
1501                            NETIF_F_HW_VLAN_RX |
1502                            NETIF_F_HW_VLAN_FILTER;
1503
1504         netdev->features |= NETIF_F_IPV6_CSUM;
1505         netdev->features |= NETIF_F_TSO;
1506         netdev->features |= NETIF_F_TSO6;
1507         netdev->features |= NETIF_F_GRO;
1508
1509         netdev->vlan_features |= NETIF_F_TSO;
1510         netdev->vlan_features |= NETIF_F_TSO6;
1511         netdev->vlan_features |= NETIF_F_IP_CSUM;
1512         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1513         netdev->vlan_features |= NETIF_F_SG;
1514
1515         if (pci_using_dac)
1516                 netdev->features |= NETIF_F_HIGHDMA;
1517
1518         if (hw->mac.type >= e1000_82576)
1519                 netdev->features |= NETIF_F_SCTP_CSUM;
1520
1521         adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1522
1523         /* before reading the NVM, reset the controller to put the device in a
1524          * known good starting state */
1525         hw->mac.ops.reset_hw(hw);
1526
1527         /* make sure the NVM is good */
1528         if (igb_validate_nvm_checksum(hw) < 0) {
1529                 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1530                 err = -EIO;
1531                 goto err_eeprom;
1532         }
1533
1534         /* copy the MAC address out of the NVM */
1535         if (hw->mac.ops.read_mac_addr(hw))
1536                 dev_err(&pdev->dev, "NVM Read Error\n");
1537
1538         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1539         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1540
1541         if (!is_valid_ether_addr(netdev->perm_addr)) {
1542                 dev_err(&pdev->dev, "Invalid MAC Address\n");
1543                 err = -EIO;
1544                 goto err_eeprom;
1545         }
1546
1547         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1548                     (unsigned long) adapter);
1549         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1550                     (unsigned long) adapter);
1551
1552         INIT_WORK(&adapter->reset_task, igb_reset_task);
1553         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1554
1555         /* Initialize link properties that are user-changeable */
1556         adapter->fc_autoneg = true;
1557         hw->mac.autoneg = true;
1558         hw->phy.autoneg_advertised = 0x2f;
1559
1560         hw->fc.requested_mode = e1000_fc_default;
1561         hw->fc.current_mode = e1000_fc_default;
1562
1563         igb_validate_mdi_setting(hw);
1564
1565         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1566          * enable the ACPI Magic Packet filter
1567          */
1568
1569         if (hw->bus.func == 0)
1570                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1571         else if (hw->mac.type == e1000_82580)
1572                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1573                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1574                                  &eeprom_data);
1575         else if (hw->bus.func == 1)
1576                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1577
1578         if (eeprom_data & eeprom_apme_mask)
1579                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1580
1581         /* now that we have the eeprom settings, apply the special cases where
1582          * the eeprom may be wrong or the board simply won't support wake on
1583          * lan on a particular port */
1584         switch (pdev->device) {
1585         case E1000_DEV_ID_82575GB_QUAD_COPPER:
1586                 adapter->eeprom_wol = 0;
1587                 break;
1588         case E1000_DEV_ID_82575EB_FIBER_SERDES:
1589         case E1000_DEV_ID_82576_FIBER:
1590         case E1000_DEV_ID_82576_SERDES:
1591                 /* Wake events only supported on port A for dual fiber
1592                  * regardless of eeprom setting */
1593                 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1594                         adapter->eeprom_wol = 0;
1595                 break;
1596         case E1000_DEV_ID_82576_QUAD_COPPER:
1597                 /* if quad port adapter, disable WoL on all but port A */
1598                 if (global_quad_port_a != 0)
1599                         adapter->eeprom_wol = 0;
1600                 else
1601                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1602                 /* Reset for multiple quad port adapters */
1603                 if (++global_quad_port_a == 4)
1604                         global_quad_port_a = 0;
1605                 break;
1606         }
1607
1608         /* initialize the wol settings based on the eeprom settings */
1609         adapter->wol = adapter->eeprom_wol;
1610         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1611
1612         /* reset the hardware with the new settings */
1613         igb_reset(adapter);
1614
1615         /* let the f/w know that the h/w is now under the control of the
1616          * driver. */
1617         igb_get_hw_control(adapter);
1618
1619         strcpy(netdev->name, "eth%d");
1620         err = register_netdev(netdev);
1621         if (err)
1622                 goto err_register;
1623
1624         /* carrier off reporting is important to ethtool even BEFORE open */
1625         netif_carrier_off(netdev);
1626
1627 #ifdef CONFIG_IGB_DCA
1628         if (dca_add_requester(&pdev->dev) == 0) {
1629                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1630                 dev_info(&pdev->dev, "DCA enabled\n");
1631                 igb_setup_dca(adapter);
1632         }
1633
1634 #endif
1635         dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1636         /* print bus type/speed/width info */
1637         dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1638                  netdev->name,
1639                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1640                                                             "unknown"),
1641                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1642                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1643                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1644                    "unknown"),
1645                  netdev->dev_addr);
1646
1647         igb_read_part_num(hw, &part_num);
1648         dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1649                 (part_num >> 8), (part_num & 0xff));
1650
1651         dev_info(&pdev->dev,
1652                 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1653                 adapter->msix_entries ? "MSI-X" :
1654                 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
1655                 adapter->num_rx_queues, adapter->num_tx_queues);
1656
1657         return 0;
1658
1659 err_register:
1660         igb_release_hw_control(adapter);
1661 err_eeprom:
1662         if (!igb_check_reset_block(hw))
1663                 igb_reset_phy(hw);
1664
1665         if (hw->flash_address)
1666                 iounmap(hw->flash_address);
1667 err_sw_init:
1668         igb_clear_interrupt_scheme(adapter);
1669         iounmap(hw->hw_addr);
1670 err_ioremap:
1671         free_netdev(netdev);
1672 err_alloc_etherdev:
1673         pci_release_selected_regions(pdev,
1674                                      pci_select_bars(pdev, IORESOURCE_MEM));
1675 err_pci_reg:
1676 err_dma:
1677         pci_disable_device(pdev);
1678         return err;
1679 }
1680
1681 /**
1682  * igb_remove - Device Removal Routine
1683  * @pdev: PCI device information struct
1684  *
1685  * igb_remove is called by the PCI subsystem to alert the driver
1686  * that it should release a PCI device.  The could be caused by a
1687  * Hot-Plug event, or because the driver is going to be removed from
1688  * memory.
1689  **/
1690 static void __devexit igb_remove(struct pci_dev *pdev)
1691 {
1692         struct net_device *netdev = pci_get_drvdata(pdev);
1693         struct igb_adapter *adapter = netdev_priv(netdev);
1694         struct e1000_hw *hw = &adapter->hw;
1695
1696         /* flush_scheduled work may reschedule our watchdog task, so
1697          * explicitly disable watchdog tasks from being rescheduled  */
1698         set_bit(__IGB_DOWN, &adapter->state);
1699         del_timer_sync(&adapter->watchdog_timer);
1700         del_timer_sync(&adapter->phy_info_timer);
1701
1702         flush_scheduled_work();
1703
1704 #ifdef CONFIG_IGB_DCA
1705         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
1706                 dev_info(&pdev->dev, "DCA disabled\n");
1707                 dca_remove_requester(&pdev->dev);
1708                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
1709                 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
1710         }
1711 #endif
1712
1713         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
1714          * would have already happened in close and is redundant. */
1715         igb_release_hw_control(adapter);
1716
1717         unregister_netdev(netdev);
1718
1719         if (!igb_check_reset_block(hw))
1720                 igb_reset_phy(hw);
1721
1722         igb_clear_interrupt_scheme(adapter);
1723
1724 #ifdef CONFIG_PCI_IOV
1725         /* reclaim resources allocated to VFs */
1726         if (adapter->vf_data) {
1727                 /* disable iov and allow time for transactions to clear */
1728                 pci_disable_sriov(pdev);
1729                 msleep(500);
1730
1731                 kfree(adapter->vf_data);
1732                 adapter->vf_data = NULL;
1733                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1734                 msleep(100);
1735                 dev_info(&pdev->dev, "IOV Disabled\n");
1736         }
1737 #endif
1738
1739         iounmap(hw->hw_addr);
1740         if (hw->flash_address)
1741                 iounmap(hw->flash_address);
1742         pci_release_selected_regions(pdev,
1743                                      pci_select_bars(pdev, IORESOURCE_MEM));
1744
1745         free_netdev(netdev);
1746
1747         pci_disable_pcie_error_reporting(pdev);
1748
1749         pci_disable_device(pdev);
1750 }
1751
1752 /**
1753  * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1754  * @adapter: board private structure to initialize
1755  *
1756  * This function initializes the vf specific data storage and then attempts to
1757  * allocate the VFs.  The reason for ordering it this way is because it is much
1758  * mor expensive time wise to disable SR-IOV than it is to allocate and free
1759  * the memory for the VFs.
1760  **/
1761 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1762 {
1763 #ifdef CONFIG_PCI_IOV
1764         struct pci_dev *pdev = adapter->pdev;
1765
1766         if (adapter->vfs_allocated_count > 7)
1767                 adapter->vfs_allocated_count = 7;
1768
1769         if (adapter->vfs_allocated_count) {
1770                 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1771                                            sizeof(struct vf_data_storage),
1772                                            GFP_KERNEL);
1773                 /* if allocation failed then we do not support SR-IOV */
1774                 if (!adapter->vf_data) {
1775                         adapter->vfs_allocated_count = 0;
1776                         dev_err(&pdev->dev, "Unable to allocate memory for VF "
1777                                 "Data Storage\n");
1778                 }
1779         }
1780
1781         if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1782                 kfree(adapter->vf_data);
1783                 adapter->vf_data = NULL;
1784 #endif /* CONFIG_PCI_IOV */
1785                 adapter->vfs_allocated_count = 0;
1786 #ifdef CONFIG_PCI_IOV
1787         } else {
1788                 unsigned char mac_addr[ETH_ALEN];
1789                 int i;
1790                 dev_info(&pdev->dev, "%d vfs allocated\n",
1791                          adapter->vfs_allocated_count);
1792                 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1793                         random_ether_addr(mac_addr);
1794                         igb_set_vf_mac(adapter, i, mac_addr);
1795                 }
1796         }
1797 #endif /* CONFIG_PCI_IOV */
1798 }
1799
1800
1801 /**
1802  * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
1803  * @adapter: board private structure to initialize
1804  *
1805  * igb_init_hw_timer initializes the function pointer and values for the hw
1806  * timer found in hardware.
1807  **/
1808 static void igb_init_hw_timer(struct igb_adapter *adapter)
1809 {
1810         struct e1000_hw *hw = &adapter->hw;
1811
1812         switch (hw->mac.type) {
1813         case e1000_82580:
1814                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1815                 adapter->cycles.read = igb_read_clock;
1816                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1817                 adapter->cycles.mult = 1;
1818                 /*
1819                  * The 82580 timesync updates the system timer every 8ns by 8ns
1820                  * and the value cannot be shifted.  Instead we need to shift
1821                  * the registers to generate a 64bit timer value.  As a result
1822                  * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
1823                  * 24 in order to generate a larger value for synchronization.
1824                  */
1825                 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
1826                 /* disable system timer temporarily by setting bit 31 */
1827                 wr32(E1000_TSAUXC, 0x80000000);
1828                 wrfl();
1829
1830                 /* Set registers so that rollover occurs soon to test this. */
1831                 wr32(E1000_SYSTIMR, 0x00000000);
1832                 wr32(E1000_SYSTIML, 0x80000000);
1833                 wr32(E1000_SYSTIMH, 0x000000FF);
1834                 wrfl();
1835
1836                 /* enable system timer by clearing bit 31 */
1837                 wr32(E1000_TSAUXC, 0x0);
1838                 wrfl();
1839
1840                 timecounter_init(&adapter->clock,
1841                                  &adapter->cycles,
1842                                  ktime_to_ns(ktime_get_real()));
1843                 /*
1844                  * Synchronize our NIC clock against system wall clock. NIC
1845                  * time stamp reading requires ~3us per sample, each sample
1846                  * was pretty stable even under load => only require 10
1847                  * samples for each offset comparison.
1848                  */
1849                 memset(&adapter->compare, 0, sizeof(adapter->compare));
1850                 adapter->compare.source = &adapter->clock;
1851                 adapter->compare.target = ktime_get_real;
1852                 adapter->compare.num_samples = 10;
1853                 timecompare_update(&adapter->compare, 0);
1854                 break;
1855         case e1000_82576:
1856                 /*
1857                  * Initialize hardware timer: we keep it running just in case
1858                  * that some program needs it later on.
1859                  */
1860                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1861                 adapter->cycles.read = igb_read_clock;
1862                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1863                 adapter->cycles.mult = 1;
1864                 /**
1865                  * Scale the NIC clock cycle by a large factor so that
1866                  * relatively small clock corrections can be added or
1867                  * substracted at each clock tick. The drawbacks of a large
1868                  * factor are a) that the clock register overflows more quickly
1869                  * (not such a big deal) and b) that the increment per tick has
1870                  * to fit into 24 bits.  As a result we need to use a shift of
1871                  * 19 so we can fit a value of 16 into the TIMINCA register.
1872                  */
1873                 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1874                 wr32(E1000_TIMINCA,
1875                                 (1 << E1000_TIMINCA_16NS_SHIFT) |
1876                                 (16 << IGB_82576_TSYNC_SHIFT));
1877
1878                 /* Set registers so that rollover occurs soon to test this. */
1879                 wr32(E1000_SYSTIML, 0x00000000);
1880                 wr32(E1000_SYSTIMH, 0xFF800000);
1881                 wrfl();
1882
1883                 timecounter_init(&adapter->clock,
1884                                  &adapter->cycles,
1885                                  ktime_to_ns(ktime_get_real()));
1886                 /*
1887                  * Synchronize our NIC clock against system wall clock. NIC
1888                  * time stamp reading requires ~3us per sample, each sample
1889                  * was pretty stable even under load => only require 10
1890                  * samples for each offset comparison.
1891                  */
1892                 memset(&adapter->compare, 0, sizeof(adapter->compare));
1893                 adapter->compare.source = &adapter->clock;
1894                 adapter->compare.target = ktime_get_real;
1895                 adapter->compare.num_samples = 10;
1896                 timecompare_update(&adapter->compare, 0);
1897                 break;
1898         case e1000_82575:
1899                 /* 82575 does not support timesync */
1900         default:
1901                 break;
1902         }
1903
1904 }
1905
1906 /**
1907  * igb_sw_init - Initialize general software structures (struct igb_adapter)
1908  * @adapter: board private structure to initialize
1909  *
1910  * igb_sw_init initializes the Adapter private data structure.
1911  * Fields are initialized based on PCI device information and
1912  * OS network device settings (MTU size).
1913  **/
1914 static int __devinit igb_sw_init(struct igb_adapter *adapter)
1915 {
1916         struct e1000_hw *hw = &adapter->hw;
1917         struct net_device *netdev = adapter->netdev;
1918         struct pci_dev *pdev = adapter->pdev;
1919
1920         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1921
1922         adapter->tx_ring_count = IGB_DEFAULT_TXD;
1923         adapter->rx_ring_count = IGB_DEFAULT_RXD;
1924         adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1925         adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1926
1927         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1928         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1929
1930 #ifdef CONFIG_PCI_IOV
1931         if (hw->mac.type == e1000_82576)
1932                 adapter->vfs_allocated_count = max_vfs;
1933
1934 #endif /* CONFIG_PCI_IOV */
1935         adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
1936
1937         /*
1938          * if rss_queues > 4 or vfs are going to be allocated with rss_queues
1939          * then we should combine the queues into a queue pair in order to
1940          * conserve interrupts due to limited supply
1941          */
1942         if ((adapter->rss_queues > 4) ||
1943             ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
1944                 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1945
1946         /* This call may decrease the number of queues */
1947         if (igb_init_interrupt_scheme(adapter)) {
1948                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1949                 return -ENOMEM;
1950         }
1951
1952         igb_init_hw_timer(adapter);
1953         igb_probe_vfs(adapter);
1954
1955         /* Explicitly disable IRQ since the NIC can be in any state. */
1956         igb_irq_disable(adapter);
1957
1958         set_bit(__IGB_DOWN, &adapter->state);
1959         return 0;
1960 }
1961
1962 /**
1963  * igb_open - Called when a network interface is made active
1964  * @netdev: network interface device structure
1965  *
1966  * Returns 0 on success, negative value on failure
1967  *
1968  * The open entry point is called when a network interface is made
1969  * active by the system (IFF_UP).  At this point all resources needed
1970  * for transmit and receive operations are allocated, the interrupt
1971  * handler is registered with the OS, the watchdog timer is started,
1972  * and the stack is notified that the interface is ready.
1973  **/
1974 static int igb_open(struct net_device *netdev)
1975 {
1976         struct igb_adapter *adapter = netdev_priv(netdev);
1977         struct e1000_hw *hw = &adapter->hw;
1978         int err;
1979         int i;
1980
1981         /* disallow open during test */
1982         if (test_bit(__IGB_TESTING, &adapter->state))
1983                 return -EBUSY;
1984
1985         netif_carrier_off(netdev);
1986
1987         /* allocate transmit descriptors */
1988         err = igb_setup_all_tx_resources(adapter);
1989         if (err)
1990                 goto err_setup_tx;
1991
1992         /* allocate receive descriptors */
1993         err = igb_setup_all_rx_resources(adapter);
1994         if (err)
1995                 goto err_setup_rx;
1996
1997         /* e1000_power_up_phy(adapter); */
1998
1999         /* before we allocate an interrupt, we must be ready to handle it.
2000          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2001          * as soon as we call pci_request_irq, so we have to setup our
2002          * clean_rx handler before we do so.  */
2003         igb_configure(adapter);
2004
2005         err = igb_request_irq(adapter);
2006         if (err)
2007                 goto err_req_irq;
2008
2009         /* From here on the code is the same as igb_up() */
2010         clear_bit(__IGB_DOWN, &adapter->state);
2011
2012         for (i = 0; i < adapter->num_q_vectors; i++) {
2013                 struct igb_q_vector *q_vector = adapter->q_vector[i];
2014                 napi_enable(&q_vector->napi);
2015         }
2016
2017         /* Clear any pending interrupts. */
2018         rd32(E1000_ICR);
2019
2020         igb_irq_enable(adapter);
2021
2022         /* notify VFs that reset has been completed */
2023         if (adapter->vfs_allocated_count) {
2024                 u32 reg_data = rd32(E1000_CTRL_EXT);
2025                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2026                 wr32(E1000_CTRL_EXT, reg_data);
2027         }
2028
2029         netif_tx_start_all_queues(netdev);
2030
2031         /* start the watchdog. */
2032         hw->mac.get_link_status = 1;
2033         schedule_work(&adapter->watchdog_task);
2034
2035         return 0;
2036
2037 err_req_irq:
2038         igb_release_hw_control(adapter);
2039         /* e1000_power_down_phy(adapter); */
2040         igb_free_all_rx_resources(adapter);
2041 err_setup_rx:
2042         igb_free_all_tx_resources(adapter);
2043 err_setup_tx:
2044         igb_reset(adapter);
2045
2046         return err;
2047 }
2048
2049 /**
2050  * igb_close - Disables a network interface
2051  * @netdev: network interface device structure
2052  *
2053  * Returns 0, this is not allowed to fail
2054  *
2055  * The close entry point is called when an interface is de-activated
2056  * by the OS.  The hardware is still under the driver's control, but
2057  * needs to be disabled.  A global MAC reset is issued to stop the
2058  * hardware, and all transmit and receive resources are freed.
2059  **/
2060 static int igb_close(struct net_device *netdev)
2061 {
2062         struct igb_adapter *adapter = netdev_priv(netdev);
2063
2064         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2065         igb_down(adapter);
2066
2067         igb_free_irq(adapter);
2068
2069         igb_free_all_tx_resources(adapter);
2070         igb_free_all_rx_resources(adapter);
2071
2072         return 0;
2073 }
2074
2075 /**
2076  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2077  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2078  *
2079  * Return 0 on success, negative on failure
2080  **/
2081 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2082 {
2083         struct pci_dev *pdev = tx_ring->pdev;
2084         int size;
2085
2086         size = sizeof(struct igb_buffer) * tx_ring->count;
2087         tx_ring->buffer_info = vmalloc(size);
2088         if (!tx_ring->buffer_info)
2089                 goto err;
2090         memset(tx_ring->buffer_info, 0, size);
2091
2092         /* round up to nearest 4K */
2093         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2094         tx_ring->size = ALIGN(tx_ring->size, 4096);
2095
2096         tx_ring->desc = pci_alloc_consistent(pdev,
2097                                              tx_ring->size,
2098                                              &tx_ring->dma);
2099
2100         if (!tx_ring->desc)
2101                 goto err;
2102
2103         tx_ring->next_to_use = 0;
2104         tx_ring->next_to_clean = 0;
2105         return 0;
2106
2107 err:
2108         vfree(tx_ring->buffer_info);
2109         dev_err(&pdev->dev,
2110                 "Unable to allocate memory for the transmit descriptor ring\n");
2111         return -ENOMEM;
2112 }
2113
2114 /**
2115  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2116  *                                (Descriptors) for all queues
2117  * @adapter: board private structure
2118  *
2119  * Return 0 on success, negative on failure
2120  **/
2121 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2122 {
2123         struct pci_dev *pdev = adapter->pdev;
2124         int i, err = 0;
2125
2126         for (i = 0; i < adapter->num_tx_queues; i++) {
2127                 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
2128                 if (err) {
2129                         dev_err(&pdev->dev,
2130                                 "Allocation for Tx Queue %u failed\n", i);
2131                         for (i--; i >= 0; i--)
2132                                 igb_free_tx_resources(&adapter->tx_ring[i]);
2133                         break;
2134                 }
2135         }
2136
2137         for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2138                 int r_idx = i % adapter->num_tx_queues;
2139                 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
2140         }
2141         return err;
2142 }
2143
2144 /**
2145  * igb_setup_tctl - configure the transmit control registers
2146  * @adapter: Board private structure
2147  **/
2148 void igb_setup_tctl(struct igb_adapter *adapter)
2149 {
2150         struct e1000_hw *hw = &adapter->hw;
2151         u32 tctl;
2152
2153         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2154         wr32(E1000_TXDCTL(0), 0);
2155
2156         /* Program the Transmit Control Register */
2157         tctl = rd32(E1000_TCTL);
2158         tctl &= ~E1000_TCTL_CT;
2159         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2160                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2161
2162         igb_config_collision_dist(hw);
2163
2164         /* Enable transmits */
2165         tctl |= E1000_TCTL_EN;
2166
2167         wr32(E1000_TCTL, tctl);
2168 }
2169
2170 /**
2171  * igb_configure_tx_ring - Configure transmit ring after Reset
2172  * @adapter: board private structure
2173  * @ring: tx ring to configure
2174  *
2175  * Configure a transmit ring after a reset.
2176  **/
2177 void igb_configure_tx_ring(struct igb_adapter *adapter,
2178                            struct igb_ring *ring)
2179 {
2180         struct e1000_hw *hw = &adapter->hw;
2181         u32 txdctl;
2182         u64 tdba = ring->dma;
2183         int reg_idx = ring->reg_idx;
2184
2185         /* disable the queue */
2186         txdctl = rd32(E1000_TXDCTL(reg_idx));
2187         wr32(E1000_TXDCTL(reg_idx),
2188                         txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2189         wrfl();
2190         mdelay(10);
2191
2192         wr32(E1000_TDLEN(reg_idx),
2193                         ring->count * sizeof(union e1000_adv_tx_desc));
2194         wr32(E1000_TDBAL(reg_idx),
2195                         tdba & 0x00000000ffffffffULL);
2196         wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2197
2198         ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2199         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2200         writel(0, ring->head);
2201         writel(0, ring->tail);
2202
2203         txdctl |= IGB_TX_PTHRESH;
2204         txdctl |= IGB_TX_HTHRESH << 8;
2205         txdctl |= IGB_TX_WTHRESH << 16;
2206
2207         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2208         wr32(E1000_TXDCTL(reg_idx), txdctl);
2209 }
2210
2211 /**
2212  * igb_configure_tx - Configure transmit Unit after Reset
2213  * @adapter: board private structure
2214  *
2215  * Configure the Tx unit of the MAC after a reset.
2216  **/
2217 static void igb_configure_tx(struct igb_adapter *adapter)
2218 {
2219         int i;
2220
2221         for (i = 0; i < adapter->num_tx_queues; i++)
2222                 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
2223 }
2224
2225 /**
2226  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2227  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2228  *
2229  * Returns 0 on success, negative on failure
2230  **/
2231 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2232 {
2233         struct pci_dev *pdev = rx_ring->pdev;
2234         int size, desc_len;
2235
2236         size = sizeof(struct igb_buffer) * rx_ring->count;
2237         rx_ring->buffer_info = vmalloc(size);
2238         if (!rx_ring->buffer_info)
2239                 goto err;
2240         memset(rx_ring->buffer_info, 0, size);
2241
2242         desc_len = sizeof(union e1000_adv_rx_desc);
2243
2244         /* Round up to nearest 4K */
2245         rx_ring->size = rx_ring->count * desc_len;
2246         rx_ring->size = ALIGN(rx_ring->size, 4096);
2247
2248         rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2249                                              &rx_ring->dma);
2250
2251         if (!rx_ring->desc)
2252                 goto err;
2253
2254         rx_ring->next_to_clean = 0;
2255         rx_ring->next_to_use = 0;
2256
2257         return 0;
2258
2259 err:
2260         vfree(rx_ring->buffer_info);
2261         rx_ring->buffer_info = NULL;
2262         dev_err(&pdev->dev, "Unable to allocate memory for "
2263                 "the receive descriptor ring\n");
2264         return -ENOMEM;
2265 }
2266
2267 /**
2268  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2269  *                                (Descriptors) for all queues
2270  * @adapter: board private structure
2271  *
2272  * Return 0 on success, negative on failure
2273  **/
2274 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2275 {
2276         struct pci_dev *pdev = adapter->pdev;
2277         int i, err = 0;
2278
2279         for (i = 0; i < adapter->num_rx_queues; i++) {
2280                 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
2281                 if (err) {
2282                         dev_err(&pdev->dev,
2283                                 "Allocation for Rx Queue %u failed\n", i);
2284                         for (i--; i >= 0; i--)
2285                                 igb_free_rx_resources(&adapter->rx_ring[i]);
2286                         break;
2287                 }
2288         }
2289
2290         return err;
2291 }
2292
2293 /**
2294  * igb_setup_mrqc - configure the multiple receive queue control registers
2295  * @adapter: Board private structure
2296  **/
2297 static void igb_setup_mrqc(struct igb_adapter *adapter)
2298 {
2299         struct e1000_hw *hw = &adapter->hw;
2300         u32 mrqc, rxcsum;
2301         u32 j, num_rx_queues, shift = 0, shift2 = 0;
2302         union e1000_reta {
2303                 u32 dword;
2304                 u8  bytes[4];
2305         } reta;
2306         static const u8 rsshash[40] = {
2307                 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2308                 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2309                 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2310                 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2311
2312         /* Fill out hash function seeds */
2313         for (j = 0; j < 10; j++) {
2314                 u32 rsskey = rsshash[(j * 4)];
2315                 rsskey |= rsshash[(j * 4) + 1] << 8;
2316                 rsskey |= rsshash[(j * 4) + 2] << 16;
2317                 rsskey |= rsshash[(j * 4) + 3] << 24;
2318                 array_wr32(E1000_RSSRK(0), j, rsskey);
2319         }
2320
2321         num_rx_queues = adapter->rss_queues;
2322
2323         if (adapter->vfs_allocated_count) {
2324                 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2325                 switch (hw->mac.type) {
2326                 case e1000_82580:
2327                         num_rx_queues = 1;
2328                         shift = 0;
2329                         break;
2330                 case e1000_82576:
2331                         shift = 3;
2332                         num_rx_queues = 2;
2333                         break;
2334                 case e1000_82575:
2335                         shift = 2;
2336                         shift2 = 6;
2337                 default:
2338                         break;
2339                 }
2340         } else {
2341                 if (hw->mac.type == e1000_82575)
2342                         shift = 6;
2343         }
2344
2345         for (j = 0; j < (32 * 4); j++) {
2346                 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2347                 if (shift2)
2348                         reta.bytes[j & 3] |= num_rx_queues << shift2;
2349                 if ((j & 3) == 3)
2350                         wr32(E1000_RETA(j >> 2), reta.dword);
2351         }
2352
2353         /*
2354          * Disable raw packet checksumming so that RSS hash is placed in
2355          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
2356          * offloads as they are enabled by default
2357          */
2358         rxcsum = rd32(E1000_RXCSUM);
2359         rxcsum |= E1000_RXCSUM_PCSD;
2360
2361         if (adapter->hw.mac.type >= e1000_82576)
2362                 /* Enable Receive Checksum Offload for SCTP */
2363                 rxcsum |= E1000_RXCSUM_CRCOFL;
2364
2365         /* Don't need to set TUOFL or IPOFL, they default to 1 */
2366         wr32(E1000_RXCSUM, rxcsum);
2367
2368         /* If VMDq is enabled then we set the appropriate mode for that, else
2369          * we default to RSS so that an RSS hash is calculated per packet even
2370          * if we are only using one queue */
2371         if (adapter->vfs_allocated_count) {
2372                 if (hw->mac.type > e1000_82575) {
2373                         /* Set the default pool for the PF's first queue */
2374                         u32 vtctl = rd32(E1000_VT_CTL);
2375                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2376                                    E1000_VT_CTL_DISABLE_DEF_POOL);
2377                         vtctl |= adapter->vfs_allocated_count <<
2378                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2379                         wr32(E1000_VT_CTL, vtctl);
2380                 }
2381                 if (adapter->rss_queues > 1)
2382                         mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2383                 else
2384                         mrqc = E1000_MRQC_ENABLE_VMDQ;
2385         } else {
2386                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2387         }
2388         igb_vmm_control(adapter);
2389
2390         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2391                  E1000_MRQC_RSS_FIELD_IPV4_TCP);
2392         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2393                  E1000_MRQC_RSS_FIELD_IPV6_TCP);
2394         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2395                  E1000_MRQC_RSS_FIELD_IPV6_UDP);
2396         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2397                  E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2398
2399         wr32(E1000_MRQC, mrqc);
2400 }
2401
2402 /**
2403  * igb_setup_rctl - configure the receive control registers
2404  * @adapter: Board private structure
2405  **/
2406 void igb_setup_rctl(struct igb_adapter *adapter)
2407 {
2408         struct e1000_hw *hw = &adapter->hw;
2409         u32 rctl;
2410
2411         rctl = rd32(E1000_RCTL);
2412
2413         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2414         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2415
2416         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2417                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2418
2419         /*
2420          * enable stripping of CRC. It's unlikely this will break BMC
2421          * redirection as it did with e1000. Newer features require
2422          * that the HW strips the CRC.
2423          */
2424         rctl |= E1000_RCTL_SECRC;
2425
2426         /* disable store bad packets and clear size bits. */
2427         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2428
2429         /* enable LPE to prevent packets larger than max_frame_size */
2430         rctl |= E1000_RCTL_LPE;
2431
2432         /* disable queue 0 to prevent tail write w/o re-config */
2433         wr32(E1000_RXDCTL(0), 0);
2434
2435         /* Attention!!!  For SR-IOV PF driver operations you must enable
2436          * queue drop for all VF and PF queues to prevent head of line blocking
2437          * if an un-trusted VF does not provide descriptors to hardware.
2438          */
2439         if (adapter->vfs_allocated_count) {
2440                 /* set all queue drop enable bits */
2441                 wr32(E1000_QDE, ALL_QUEUES);
2442         }
2443
2444         wr32(E1000_RCTL, rctl);
2445 }
2446
2447 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2448                                    int vfn)
2449 {
2450         struct e1000_hw *hw = &adapter->hw;
2451         u32 vmolr;
2452
2453         /* if it isn't the PF check to see if VFs are enabled and
2454          * increase the size to support vlan tags */
2455         if (vfn < adapter->vfs_allocated_count &&
2456             adapter->vf_data[vfn].vlans_enabled)
2457                 size += VLAN_TAG_SIZE;
2458
2459         vmolr = rd32(E1000_VMOLR(vfn));
2460         vmolr &= ~E1000_VMOLR_RLPML_MASK;
2461         vmolr |= size | E1000_VMOLR_LPE;
2462         wr32(E1000_VMOLR(vfn), vmolr);
2463
2464         return 0;
2465 }
2466
2467 /**
2468  * igb_rlpml_set - set maximum receive packet size
2469  * @adapter: board private structure
2470  *
2471  * Configure maximum receivable packet size.
2472  **/
2473 static void igb_rlpml_set(struct igb_adapter *adapter)
2474 {
2475         u32 max_frame_size = adapter->max_frame_size;
2476         struct e1000_hw *hw = &adapter->hw;
2477         u16 pf_id = adapter->vfs_allocated_count;
2478
2479         if (adapter->vlgrp)
2480                 max_frame_size += VLAN_TAG_SIZE;
2481
2482         /* if vfs are enabled we set RLPML to the largest possible request
2483          * size and set the VMOLR RLPML to the size we need */
2484         if (pf_id) {
2485                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2486                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2487         }
2488
2489         wr32(E1000_RLPML, max_frame_size);
2490 }
2491
2492 static inline void igb_set_vmolr(struct igb_adapter *adapter, int vfn)
2493 {
2494         struct e1000_hw *hw = &adapter->hw;
2495         u32 vmolr;
2496
2497         /*
2498          * This register exists only on 82576 and newer so if we are older then
2499          * we should exit and do nothing
2500          */
2501         if (hw->mac.type < e1000_82576)
2502                 return;
2503
2504         vmolr = rd32(E1000_VMOLR(vfn));
2505         vmolr |= E1000_VMOLR_AUPE |        /* Accept untagged packets */
2506                  E1000_VMOLR_STRVLAN;      /* Strip vlan tags */
2507
2508         /* clear all bits that might not be set */
2509         vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2510
2511         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2512                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2513         /*
2514          * for VMDq only allow the VFs and pool 0 to accept broadcast and
2515          * multicast packets
2516          */
2517         if (vfn <= adapter->vfs_allocated_count)
2518                 vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
2519
2520         wr32(E1000_VMOLR(vfn), vmolr);
2521 }
2522
2523 /**
2524  * igb_configure_rx_ring - Configure a receive ring after Reset
2525  * @adapter: board private structure
2526  * @ring: receive ring to be configured
2527  *
2528  * Configure the Rx unit of the MAC after a reset.
2529  **/
2530 void igb_configure_rx_ring(struct igb_adapter *adapter,
2531                            struct igb_ring *ring)
2532 {
2533         struct e1000_hw *hw = &adapter->hw;
2534         u64 rdba = ring->dma;
2535         int reg_idx = ring->reg_idx;
2536         u32 srrctl, rxdctl;
2537
2538         /* disable the queue */
2539         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2540         wr32(E1000_RXDCTL(reg_idx),
2541                         rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2542
2543         /* Set DMA base address registers */
2544         wr32(E1000_RDBAL(reg_idx),
2545              rdba & 0x00000000ffffffffULL);
2546         wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2547         wr32(E1000_RDLEN(reg_idx),
2548                        ring->count * sizeof(union e1000_adv_rx_desc));
2549
2550         /* initialize head and tail */
2551         ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2552         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2553         writel(0, ring->head);
2554         writel(0, ring->tail);
2555
2556         /* set descriptor configuration */
2557         if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2558                 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2559                          E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2560 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2561                 srrctl |= IGB_RXBUFFER_16384 >>
2562                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2563 #else
2564                 srrctl |= (PAGE_SIZE / 2) >>
2565                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2566 #endif
2567                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2568         } else {
2569                 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2570                          E1000_SRRCTL_BSIZEPKT_SHIFT;
2571                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2572         }
2573
2574         wr32(E1000_SRRCTL(reg_idx), srrctl);
2575
2576         /* set filtering for VMDQ pools */
2577         igb_set_vmolr(adapter, reg_idx & 0x7);
2578
2579         /* enable receive descriptor fetching */
2580         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2581         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2582         rxdctl &= 0xFFF00000;
2583         rxdctl |= IGB_RX_PTHRESH;
2584         rxdctl |= IGB_RX_HTHRESH << 8;
2585         rxdctl |= IGB_RX_WTHRESH << 16;
2586         wr32(E1000_RXDCTL(reg_idx), rxdctl);
2587 }
2588
2589 /**
2590  * igb_configure_rx - Configure receive Unit after Reset
2591  * @adapter: board private structure
2592  *
2593  * Configure the Rx unit of the MAC after a reset.
2594  **/
2595 static void igb_configure_rx(struct igb_adapter *adapter)
2596 {
2597         int i;
2598
2599         /* set UTA to appropriate mode */
2600         igb_set_uta(adapter);
2601
2602         /* set the correct pool for the PF default MAC address in entry 0 */
2603         igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2604                          adapter->vfs_allocated_count);
2605
2606         /* Setup the HW Rx Head and Tail Descriptor Pointers and
2607          * the Base and Length of the Rx Descriptor Ring */
2608         for (i = 0; i < adapter->num_rx_queues; i++)
2609                 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
2610 }
2611
2612 /**
2613  * igb_free_tx_resources - Free Tx Resources per Queue
2614  * @tx_ring: Tx descriptor ring for a specific queue
2615  *
2616  * Free all transmit software resources
2617  **/
2618 void igb_free_tx_resources(struct igb_ring *tx_ring)
2619 {
2620         igb_clean_tx_ring(tx_ring);
2621
2622         vfree(tx_ring->buffer_info);
2623         tx_ring->buffer_info = NULL;
2624
2625         /* if not set, then don't free */
2626         if (!tx_ring->desc)
2627                 return;
2628
2629         pci_free_consistent(tx_ring->pdev, tx_ring->size,
2630                             tx_ring->desc, tx_ring->dma);
2631
2632         tx_ring->desc = NULL;
2633 }
2634
2635 /**
2636  * igb_free_all_tx_resources - Free Tx Resources for All Queues
2637  * @adapter: board private structure
2638  *
2639  * Free all transmit software resources
2640  **/
2641 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2642 {
2643         int i;
2644
2645         for (i = 0; i < adapter->num_tx_queues; i++)
2646                 igb_free_tx_resources(&adapter->tx_ring[i]);
2647 }
2648
2649 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2650                                     struct igb_buffer *buffer_info)
2651 {
2652         if (buffer_info->dma) {
2653                 if (buffer_info->mapped_as_page)
2654                         pci_unmap_page(tx_ring->pdev,
2655                                         buffer_info->dma,
2656                                         buffer_info->length,
2657                                         PCI_DMA_TODEVICE);
2658                 else
2659                         pci_unmap_single(tx_ring->pdev,
2660                                         buffer_info->dma,
2661                                         buffer_info->length,
2662                                         PCI_DMA_TODEVICE);
2663                 buffer_info->dma = 0;
2664         }
2665         if (buffer_info->skb) {
2666                 dev_kfree_skb_any(buffer_info->skb);
2667                 buffer_info->skb = NULL;
2668         }
2669         buffer_info->time_stamp = 0;
2670         buffer_info->length = 0;
2671         buffer_info->next_to_watch = 0;
2672         buffer_info->mapped_as_page = false;
2673 }
2674
2675 /**
2676  * igb_clean_tx_ring - Free Tx Buffers
2677  * @tx_ring: ring to be cleaned
2678  **/
2679 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
2680 {
2681         struct igb_buffer *buffer_info;
2682         unsigned long size;
2683         unsigned int i;
2684
2685         if (!tx_ring->buffer_info)
2686                 return;
2687         /* Free all the Tx ring sk_buffs */
2688
2689         for (i = 0; i < tx_ring->count; i++) {
2690                 buffer_info = &tx_ring->buffer_info[i];
2691                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
2692         }
2693
2694         size = sizeof(struct igb_buffer) * tx_ring->count;
2695         memset(tx_ring->buffer_info, 0, size);
2696
2697         /* Zero out the descriptor ring */
2698         memset(tx_ring->desc, 0, tx_ring->size);
2699
2700         tx_ring->next_to_use = 0;
2701         tx_ring->next_to_clean = 0;
2702 }
2703
2704 /**
2705  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2706  * @adapter: board private structure
2707  **/
2708 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2709 {
2710         int i;
2711
2712         for (i = 0; i < adapter->num_tx_queues; i++)
2713                 igb_clean_tx_ring(&adapter->tx_ring[i]);
2714 }
2715
2716 /**
2717  * igb_free_rx_resources - Free Rx Resources
2718  * @rx_ring: ring to clean the resources from
2719  *
2720  * Free all receive software resources
2721  **/
2722 void igb_free_rx_resources(struct igb_ring *rx_ring)
2723 {
2724         igb_clean_rx_ring(rx_ring);
2725
2726         vfree(rx_ring->buffer_info);
2727         rx_ring->buffer_info = NULL;
2728
2729         /* if not set, then don't free */
2730         if (!rx_ring->desc)
2731                 return;
2732
2733         pci_free_consistent(rx_ring->pdev, rx_ring->size,
2734                             rx_ring->desc, rx_ring->dma);
2735
2736         rx_ring->desc = NULL;
2737 }
2738
2739 /**
2740  * igb_free_all_rx_resources - Free Rx Resources for All Queues
2741  * @adapter: board private structure
2742  *
2743  * Free all receive software resources
2744  **/
2745 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2746 {
2747         int i;
2748
2749         for (i = 0; i < adapter->num_rx_queues; i++)
2750                 igb_free_rx_resources(&adapter->rx_ring[i]);
2751 }
2752
2753 /**
2754  * igb_clean_rx_ring - Free Rx Buffers per Queue
2755  * @rx_ring: ring to free buffers from
2756  **/
2757 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
2758 {
2759         struct igb_buffer *buffer_info;
2760         unsigned long size;
2761         unsigned int i;
2762
2763         if (!rx_ring->buffer_info)
2764                 return;
2765
2766         /* Free all the Rx ring sk_buffs */
2767         for (i = 0; i < rx_ring->count; i++) {
2768                 buffer_info = &rx_ring->buffer_info[i];
2769                 if (buffer_info->dma) {
2770                         pci_unmap_single(rx_ring->pdev,
2771                                          buffer_info->dma,
2772                                          rx_ring->rx_buffer_len,
2773                                          PCI_DMA_FROMDEVICE);
2774                         buffer_info->dma = 0;
2775                 }
2776
2777                 if (buffer_info->skb) {
2778                         dev_kfree_skb(buffer_info->skb);
2779                         buffer_info->skb = NULL;
2780                 }
2781                 if (buffer_info->page_dma) {
2782                         pci_unmap_page(rx_ring->pdev,
2783                                        buffer_info->page_dma,
2784                                        PAGE_SIZE / 2,
2785                                        PCI_DMA_FROMDEVICE);
2786                         buffer_info->page_dma = 0;
2787                 }
2788                 if (buffer_info->page) {
2789                         put_page(buffer_info->page);
2790                         buffer_info->page = NULL;
2791                         buffer_info->page_offset = 0;
2792                 }
2793         }
2794
2795         size = sizeof(struct igb_buffer) * rx_ring->count;
2796         memset(rx_ring->buffer_info, 0, size);
2797
2798         /* Zero out the descriptor ring */
2799         memset(rx_ring->desc, 0, rx_ring->size);
2800
2801         rx_ring->next_to_clean = 0;
2802         rx_ring->next_to_use = 0;
2803 }
2804
2805 /**
2806  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2807  * @adapter: board private structure
2808  **/
2809 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2810 {
2811         int i;
2812
2813         for (i = 0; i < adapter->num_rx_queues; i++)
2814                 igb_clean_rx_ring(&adapter->rx_ring[i]);
2815 }
2816
2817 /**
2818  * igb_set_mac - Change the Ethernet Address of the NIC
2819  * @netdev: network interface device structure
2820  * @p: pointer to an address structure
2821  *
2822  * Returns 0 on success, negative on failure
2823  **/
2824 static int igb_set_mac(struct net_device *netdev, void *p)
2825 {
2826         struct igb_adapter *adapter = netdev_priv(netdev);
2827         struct e1000_hw *hw = &adapter->hw;
2828         struct sockaddr *addr = p;
2829
2830         if (!is_valid_ether_addr(addr->sa_data))
2831                 return -EADDRNOTAVAIL;
2832
2833         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2834         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2835
2836         /* set the correct pool for the new PF MAC address in entry 0 */
2837         igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2838                          adapter->vfs_allocated_count);
2839
2840         return 0;
2841 }
2842
2843 /**
2844  * igb_write_mc_addr_list - write multicast addresses to MTA
2845  * @netdev: network interface device structure
2846  *
2847  * Writes multicast address list to the MTA hash table.
2848  * Returns: -ENOMEM on failure
2849  *                0 on no addresses written
2850  *                X on writing X addresses to MTA
2851  **/
2852 static int igb_write_mc_addr_list(struct net_device *netdev)
2853 {
2854         struct igb_adapter *adapter = netdev_priv(netdev);
2855         struct e1000_hw *hw = &adapter->hw;
2856         struct dev_mc_list *mc_ptr = netdev->mc_list;
2857         u8  *mta_list;
2858         u32 vmolr = 0;
2859         int i;
2860
2861         if (!netdev->mc_count) {
2862                 /* nothing to program, so clear mc list */
2863                 igb_update_mc_addr_list(hw, NULL, 0);
2864                 igb_restore_vf_multicasts(adapter);
2865                 return 0;
2866         }
2867
2868         mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2869         if (!mta_list)
2870                 return -ENOMEM;
2871
2872         /* set vmolr receive overflow multicast bit */
2873         vmolr |= E1000_VMOLR_ROMPE;
2874
2875         /* The shared function expects a packed array of only addresses. */
2876         mc_ptr = netdev->mc_list;
2877
2878         for (i = 0; i < netdev->mc_count; i++) {
2879                 if (!mc_ptr)
2880                         break;
2881                 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2882                 mc_ptr = mc_ptr->next;
2883         }
2884         igb_update_mc_addr_list(hw, mta_list, i);
2885         kfree(mta_list);
2886
2887         return netdev->mc_count;
2888 }
2889
2890 /**
2891  * igb_write_uc_addr_list - write unicast addresses to RAR table
2892  * @netdev: network interface device structure
2893  *
2894  * Writes unicast address list to the RAR table.
2895  * Returns: -ENOMEM on failure/insufficient address space
2896  *                0 on no addresses written
2897  *                X on writing X addresses to the RAR table
2898  **/
2899 static int igb_write_uc_addr_list(struct net_device *netdev)
2900 {
2901         struct igb_adapter *adapter = netdev_priv(netdev);
2902         struct e1000_hw *hw = &adapter->hw;
2903         unsigned int vfn = adapter->vfs_allocated_count;
2904         unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2905         int count = 0;
2906
2907         /* return ENOMEM indicating insufficient memory for addresses */
2908         if (netdev_uc_count(netdev) > rar_entries)
2909                 return -ENOMEM;
2910
2911         if (!netdev_uc_empty(netdev) && rar_entries) {
2912                 struct netdev_hw_addr *ha;
2913
2914                 netdev_for_each_uc_addr(ha, netdev) {
2915                         if (!rar_entries)
2916                                 break;
2917                         igb_rar_set_qsel(adapter, ha->addr,
2918                                          rar_entries--,
2919                                          vfn);
2920                         count++;
2921                 }
2922         }
2923         /* write the addresses in reverse order to avoid write combining */
2924         for (; rar_entries > 0 ; rar_entries--) {
2925                 wr32(E1000_RAH(rar_entries), 0);
2926                 wr32(E1000_RAL(rar_entries), 0);
2927         }
2928         wrfl();
2929
2930         return count;
2931 }
2932
2933 /**
2934  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2935  * @netdev: network interface device structure
2936  *
2937  * The set_rx_mode entry point is called whenever the unicast or multicast
2938  * address lists or the network interface flags are updated.  This routine is
2939  * responsible for configuring the hardware for proper unicast, multicast,
2940  * promiscuous mode, and all-multi behavior.
2941  **/
2942 static void igb_set_rx_mode(struct net_device *netdev)
2943 {
2944         struct igb_adapter *adapter = netdev_priv(netdev);
2945         struct e1000_hw *hw = &adapter->hw;
2946         unsigned int vfn = adapter->vfs_allocated_count;
2947         u32 rctl, vmolr = 0;
2948         int count;
2949
2950         /* Check for Promiscuous and All Multicast modes */
2951         rctl = rd32(E1000_RCTL);
2952
2953         /* clear the effected bits */
2954         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2955
2956         if (netdev->flags & IFF_PROMISC) {
2957                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2958                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2959         } else {
2960                 if (netdev->flags & IFF_ALLMULTI) {
2961                         rctl |= E1000_RCTL_MPE;
2962                         vmolr |= E1000_VMOLR_MPME;
2963                 } else {
2964                         /*
2965                          * Write addresses to the MTA, if the attempt fails
2966                          * then we should just turn on promiscous mode so
2967                          * that we can at least receive multicast traffic
2968                          */
2969                         count = igb_write_mc_addr_list(netdev);
2970                         if (count < 0) {
2971                                 rctl |= E1000_RCTL_MPE;
2972                                 vmolr |= E1000_VMOLR_MPME;
2973                         } else if (count) {
2974                                 vmolr |= E1000_VMOLR_ROMPE;
2975                         }
2976                 }
2977                 /*
2978                  * Write addresses to available RAR registers, if there is not
2979                  * sufficient space to store all the addresses then enable
2980                  * unicast promiscous mode
2981                  */
2982                 count = igb_write_uc_addr_list(netdev);
2983                 if (count < 0) {
2984                         rctl |= E1000_RCTL_UPE;
2985                         vmolr |= E1000_VMOLR_ROPE;
2986                 }
2987                 rctl |= E1000_RCTL_VFE;
2988         }
2989         wr32(E1000_RCTL, rctl);
2990
2991         /*
2992          * In order to support SR-IOV and eventually VMDq it is necessary to set
2993          * the VMOLR to enable the appropriate modes.  Without this workaround
2994          * we will have issues with VLAN tag stripping not being done for frames
2995          * that are only arriving because we are the default pool
2996          */
2997         if (hw->mac.type < e1000_82576)
2998                 return;
2999
3000         vmolr |= rd32(E1000_VMOLR(vfn)) &
3001                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3002         wr32(E1000_VMOLR(vfn), vmolr);
3003         igb_restore_vf_multicasts(adapter);
3004 }
3005
3006 /* Need to wait a few seconds after link up to get diagnostic information from
3007  * the phy */
3008 static void igb_update_phy_info(unsigned long data)
3009 {
3010         struct igb_adapter *adapter = (struct igb_adapter *) data;
3011         igb_get_phy_info(&adapter->hw);
3012 }
3013
3014 /**
3015  * igb_has_link - check shared code for link and determine up/down
3016  * @adapter: pointer to driver private info
3017  **/
3018 static bool igb_has_link(struct igb_adapter *adapter)
3019 {
3020         struct e1000_hw *hw = &adapter->hw;
3021         bool link_active = false;
3022         s32 ret_val = 0;
3023
3024         /* get_link_status is set on LSC (link status) interrupt or
3025          * rx sequence error interrupt.  get_link_status will stay
3026          * false until the e1000_check_for_link establishes link
3027          * for copper adapters ONLY
3028          */
3029         switch (hw->phy.media_type) {
3030         case e1000_media_type_copper:
3031                 if (hw->mac.get_link_status) {
3032                         ret_val = hw->mac.ops.check_for_link(hw);
3033                         link_active = !hw->mac.get_link_status;
3034                 } else {
3035                         link_active = true;
3036                 }
3037                 break;
3038         case e1000_media_type_internal_serdes:
3039                 ret_val = hw->mac.ops.check_for_link(hw);
3040                 link_active = hw->mac.serdes_has_link;
3041                 break;
3042         default:
3043         case e1000_media_type_unknown:
3044                 break;
3045         }
3046
3047         return link_active;
3048 }
3049
3050 /**
3051  * igb_watchdog - Timer Call-back
3052  * @data: pointer to adapter cast into an unsigned long
3053  **/
3054 static void igb_watchdog(unsigned long data)
3055 {
3056         struct igb_adapter *adapter = (struct igb_adapter *)data;
3057         /* Do the rest outside of interrupt context */
3058         schedule_work(&adapter->watchdog_task);
3059 }
3060
3061 static void igb_watchdog_task(struct work_struct *work)
3062 {
3063         struct igb_adapter *adapter = container_of(work,
3064                                                    struct igb_adapter,
3065                                                    watchdog_task);
3066         struct e1000_hw *hw = &adapter->hw;
3067         struct net_device *netdev = adapter->netdev;
3068         u32 link;
3069         int i;
3070
3071         link = igb_has_link(adapter);
3072         if (link) {
3073                 if (!netif_carrier_ok(netdev)) {
3074                         u32 ctrl;
3075                         hw->mac.ops.get_speed_and_duplex(hw,
3076                                                          &adapter->link_speed,
3077                                                          &adapter->link_duplex);
3078
3079                         ctrl = rd32(E1000_CTRL);
3080                         /* Links status message must follow this format */
3081                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3082                                  "Flow Control: %s\n",
3083                                netdev->name,
3084                                adapter->link_speed,
3085                                adapter->link_duplex == FULL_DUPLEX ?
3086                                  "Full Duplex" : "Half Duplex",
3087                                ((ctrl & E1000_CTRL_TFCE) &&
3088                                 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3089                                ((ctrl & E1000_CTRL_RFCE) ?  "RX" :
3090                                ((ctrl & E1000_CTRL_TFCE) ?  "TX" : "None")));
3091
3092                         /* tweak tx_queue_len according to speed/duplex and
3093                          * adjust the timeout factor */
3094                         netdev->tx_queue_len = adapter->tx_queue_len;
3095                         adapter->tx_timeout_factor = 1;
3096                         switch (adapter->link_speed) {
3097                         case SPEED_10:
3098                                 netdev->tx_queue_len = 10;
3099                                 adapter->tx_timeout_factor = 14;
3100                                 break;
3101                         case SPEED_100:
3102                                 netdev->tx_queue_len = 100;
3103                                 /* maybe add some timeout factor ? */
3104                                 break;
3105                         }
3106
3107                         netif_carrier_on(netdev);
3108
3109                         igb_ping_all_vfs(adapter);
3110
3111                         /* link state has changed, schedule phy info update */
3112                         if (!test_bit(__IGB_DOWN, &adapter->state))
3113                                 mod_timer(&adapter->phy_info_timer,
3114                                           round_jiffies(jiffies + 2 * HZ));
3115                 }
3116         } else {
3117                 if (netif_carrier_ok(netdev)) {
3118                         adapter->link_speed = 0;
3119                         adapter->link_duplex = 0;
3120                         /* Links status message must follow this format */
3121                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
3122                                netdev->name);
3123                         netif_carrier_off(netdev);
3124
3125                         igb_ping_all_vfs(adapter);
3126
3127                         /* link state has changed, schedule phy info update */
3128                         if (!test_bit(__IGB_DOWN, &adapter->state))
3129                                 mod_timer(&adapter->phy_info_timer,
3130                                           round_jiffies(jiffies + 2 * HZ));
3131                 }
3132         }
3133
3134         igb_update_stats(adapter);
3135         igb_update_adaptive(hw);
3136
3137         for (i = 0; i < adapter->num_tx_queues; i++) {
3138                 struct igb_ring *tx_ring = &adapter->tx_ring[i];
3139                 if (!netif_carrier_ok(netdev)) {
3140                         /* We've lost link, so the controller stops DMA,
3141                          * but we've got queued Tx work that's never going
3142                          * to get done, so reset controller to flush Tx.
3143                          * (Do the reset outside of interrupt context). */
3144                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3145                                 adapter->tx_timeout_count++;
3146                                 schedule_work(&adapter->reset_task);
3147                                 /* return immediately since reset is imminent */
3148                                 return;
3149                         }
3150                 }
3151
3152                 /* Force detection of hung controller every watchdog period */
3153                 tx_ring->detect_tx_hung = true;
3154         }
3155
3156         /* Cause software interrupt to ensure rx ring is cleaned */
3157         if (adapter->msix_entries) {
3158                 u32 eics = 0;
3159                 for (i = 0; i < adapter->num_q_vectors; i++) {
3160                         struct igb_q_vector *q_vector = adapter->q_vector[i];
3161                         eics |= q_vector->eims_value;
3162                 }
3163                 wr32(E1000_EICS, eics);
3164         } else {
3165                 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3166         }
3167
3168         /* Reset the timer */
3169         if (!test_bit(__IGB_DOWN, &adapter->state))
3170                 mod_timer(&adapter->watchdog_timer,
3171                           round_jiffies(jiffies + 2 * HZ));
3172 }
3173
3174 enum latency_range {
3175         lowest_latency = 0,
3176         low_latency = 1,
3177         bulk_latency = 2,
3178         latency_invalid = 255
3179 };
3180
3181 /**
3182  * igb_update_ring_itr - update the dynamic ITR value based on packet size
3183  *
3184  *      Stores a new ITR value based on strictly on packet size.  This
3185  *      algorithm is less sophisticated than that used in igb_update_itr,
3186  *      due to the difficulty of synchronizing statistics across multiple
3187  *      receive rings.  The divisors and thresholds used by this fuction
3188  *      were determined based on theoretical maximum wire speed and testing
3189  *      data, in order to minimize response time while increasing bulk
3190  *      throughput.
3191  *      This functionality is controlled by the InterruptThrottleRate module
3192  *      parameter (see igb_param.c)
3193  *      NOTE:  This function is called only when operating in a multiqueue
3194  *             receive environment.
3195  * @q_vector: pointer to q_vector
3196  **/
3197 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3198 {
3199         int new_val = q_vector->itr_val;
3200         int avg_wire_size = 0;
3201         struct igb_adapter *adapter = q_vector->adapter;
3202
3203         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3204          * ints/sec - ITR timer value of 120 ticks.
3205          */
3206         if (adapter->link_speed != SPEED_1000) {
3207                 new_val = 976;
3208                 goto set_itr_val;
3209         }
3210
3211         if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3212                 struct igb_ring *ring = q_vector->rx_ring;
3213                 avg_wire_size = ring->total_bytes / ring->total_packets;
3214         }
3215
3216         if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3217                 struct igb_ring *ring = q_vector->tx_ring;
3218                 avg_wire_size = max_t(u32, avg_wire_size,
3219                                       (ring->total_bytes /
3220                                        ring->total_packets));
3221         }
3222
3223         /* if avg_wire_size isn't set no work was done */
3224         if (!avg_wire_size)
3225                 goto clear_counts;
3226
3227         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3228         avg_wire_size += 24;
3229
3230         /* Don't starve jumbo frames */
3231         avg_wire_size = min(avg_wire_size, 3000);
3232
3233         /* Give a little boost to mid-size frames */
3234         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3235                 new_val = avg_wire_size / 3;
3236         else
3237                 new_val = avg_wire_size / 2;
3238
3239 set_itr_val:
3240         if (new_val != q_vector->itr_val) {
3241                 q_vector->itr_val = new_val;
3242                 q_vector->set_itr = 1;
3243         }
3244 clear_counts:
3245         if (q_vector->rx_ring) {
3246                 q_vector->rx_ring->total_bytes = 0;
3247                 q_vector->rx_ring->total_packets = 0;
3248         }
3249         if (q_vector->tx_ring) {
3250                 q_vector->tx_ring->total_bytes = 0;
3251                 q_vector->tx_ring->total_packets = 0;
3252         }
3253 }
3254
3255 /**
3256  * igb_update_itr - update the dynamic ITR value based on statistics
3257  *      Stores a new ITR value based on packets and byte
3258  *      counts during the last interrupt.  The advantage of per interrupt
3259  *      computation is faster updates and more accurate ITR for the current
3260  *      traffic pattern.  Constants in this function were computed
3261  *      based on theoretical maximum wire speed and thresholds were set based
3262  *      on testing data as well as attempting to minimize response time
3263  *      while increasing bulk throughput.
3264  *      this functionality is controlled by the InterruptThrottleRate module
3265  *      parameter (see igb_param.c)
3266  *      NOTE:  These calculations are only valid when operating in a single-
3267  *             queue environment.
3268  * @adapter: pointer to adapter
3269  * @itr_setting: current q_vector->itr_val
3270  * @packets: the number of packets during this measurement interval
3271  * @bytes: the number of bytes during this measurement interval
3272  **/
3273 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3274                                    int packets, int bytes)
3275 {
3276         unsigned int retval = itr_setting;
3277
3278         if (packets == 0)
3279                 goto update_itr_done;
3280
3281         switch (itr_setting) {
3282         case lowest_latency:
3283                 /* handle TSO and jumbo frames */
3284                 if (bytes/packets > 8000)
3285                         retval = bulk_latency;
3286                 else if ((packets < 5) && (bytes > 512))
3287                         retval = low_latency;
3288                 break;
3289         case low_latency:  /* 50 usec aka 20000 ints/s */
3290                 if (bytes > 10000) {
3291                         /* this if handles the TSO accounting */
3292                         if (bytes/packets > 8000) {
3293                                 retval = bulk_latency;
3294                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3295                                 retval = bulk_latency;
3296                         } else if ((packets > 35)) {
3297                                 retval = lowest_latency;
3298                         }
3299                 } else if (bytes/packets > 2000) {
3300                         retval = bulk_latency;
3301                 } else if (packets <= 2 && bytes < 512) {
3302                         retval = lowest_latency;
3303                 }
3304                 break;
3305         case bulk_latency: /* 250 usec aka 4000 ints/s */
3306                 if (bytes > 25000) {
3307                         if (packets > 35)
3308                                 retval = low_latency;
3309                 } else if (bytes < 1500) {
3310                         retval = low_latency;
3311                 }
3312                 break;
3313         }
3314
3315 update_itr_done:
3316         return retval;
3317 }
3318
3319 static void igb_set_itr(struct igb_adapter *adapter)
3320 {
3321         struct igb_q_vector *q_vector = adapter->q_vector[0];
3322         u16 current_itr;
3323         u32 new_itr = q_vector->itr_val;
3324
3325         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3326         if (adapter->link_speed != SPEED_1000) {
3327                 current_itr = 0;
3328                 new_itr = 4000;
3329                 goto set_itr_now;
3330         }
3331
3332         adapter->rx_itr = igb_update_itr(adapter,
3333                                     adapter->rx_itr,
3334                                     adapter->rx_ring->total_packets,
3335                                     adapter->rx_ring->total_bytes);
3336
3337         adapter->tx_itr = igb_update_itr(adapter,
3338                                     adapter->tx_itr,
3339                                     adapter->tx_ring->total_packets,
3340                                     adapter->tx_ring->total_bytes);
3341         current_itr = max(adapter->rx_itr, adapter->tx_itr);
3342
3343         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3344         if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3345                 current_itr = low_latency;
3346
3347         switch (current_itr) {
3348         /* counts and packets in update_itr are dependent on these numbers */
3349         case lowest_latency:
3350                 new_itr = 56;  /* aka 70,000 ints/sec */
3351                 break;
3352         case low_latency:
3353                 new_itr = 196; /* aka 20,000 ints/sec */
3354                 break;
3355         case bulk_latency:
3356                 new_itr = 980; /* aka 4,000 ints/sec */
3357                 break;
3358         default:
3359                 break;
3360         }
3361
3362 set_itr_now:
3363         adapter->rx_ring->total_bytes = 0;
3364         adapter->rx_ring->total_packets = 0;
3365         adapter->tx_ring->total_bytes = 0;
3366         adapter->tx_ring->total_packets = 0;
3367
3368         if (new_itr != q_vector->itr_val) {
3369                 /* this attempts to bias the interrupt rate towards Bulk
3370                  * by adding intermediate steps when interrupt rate is
3371                  * increasing */
3372                 new_itr = new_itr > q_vector->itr_val ?
3373                              max((new_itr * q_vector->itr_val) /
3374                                  (new_itr + (q_vector->itr_val >> 2)),
3375                                  new_itr) :
3376                              new_itr;
3377                 /* Don't write the value here; it resets the adapter's
3378                  * internal timer, and causes us to delay far longer than
3379                  * we should between interrupts.  Instead, we write the ITR
3380                  * value at the beginning of the next interrupt so the timing
3381                  * ends up being correct.
3382                  */
3383                 q_vector->itr_val = new_itr;
3384                 q_vector->set_itr = 1;
3385         }
3386
3387         return;
3388 }
3389
3390 #define IGB_TX_FLAGS_CSUM               0x00000001
3391 #define IGB_TX_FLAGS_VLAN               0x00000002
3392 #define IGB_TX_FLAGS_TSO                0x00000004
3393 #define IGB_TX_FLAGS_IPV4               0x00000008
3394 #define IGB_TX_FLAGS_TSTAMP             0x00000010
3395 #define IGB_TX_FLAGS_VLAN_MASK          0xffff0000
3396 #define IGB_TX_FLAGS_VLAN_SHIFT                 16
3397
3398 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3399                               struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3400 {
3401         struct e1000_adv_tx_context_desc *context_desc;
3402         unsigned int i;
3403         int err;
3404         struct igb_buffer *buffer_info;
3405         u32 info = 0, tu_cmd = 0;
3406         u32 mss_l4len_idx, l4len;
3407         *hdr_len = 0;
3408
3409         if (skb_header_cloned(skb)) {
3410                 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3411                 if (err)
3412                         return err;
3413         }
3414
3415         l4len = tcp_hdrlen(skb);
3416         *hdr_len += l4len;
3417
3418         if (skb->protocol == htons(ETH_P_IP)) {
3419                 struct iphdr *iph = ip_hdr(skb);
3420                 iph->tot_len = 0;
3421                 iph->check = 0;
3422                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3423                                                          iph->daddr, 0,
3424                                                          IPPROTO_TCP,
3425                                                          0);
3426         } else if (skb_is_gso_v6(skb)) {
3427                 ipv6_hdr(skb)->payload_len = 0;
3428                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3429                                                        &ipv6_hdr(skb)->daddr,
3430                                                        0, IPPROTO_TCP, 0);
3431         }
3432
3433         i = tx_ring->next_to_use;
3434
3435         buffer_info = &tx_ring->buffer_info[i];
3436         context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3437         /* VLAN MACLEN IPLEN */
3438         if (tx_flags & IGB_TX_FLAGS_VLAN)
3439                 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3440         info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3441         *hdr_len += skb_network_offset(skb);
3442         info |= skb_network_header_len(skb);
3443         *hdr_len += skb_network_header_len(skb);
3444         context_desc->vlan_macip_lens = cpu_to_le32(info);
3445
3446         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3447         tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3448
3449         if (skb->protocol == htons(ETH_P_IP))
3450                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3451         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3452
3453         context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3454
3455         /* MSS L4LEN IDX */
3456         mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3457         mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3458
3459         /* For 82575, context index must be unique per ring. */
3460         if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3461                 mss_l4len_idx |= tx_ring->reg_idx << 4;
3462
3463         context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3464         context_desc->seqnum_seed = 0;
3465
3466         buffer_info->time_stamp = jiffies;
3467         buffer_info->next_to_watch = i;
3468         buffer_info->dma = 0;
3469         i++;
3470         if (i == tx_ring->count)
3471                 i = 0;
3472
3473         tx_ring->next_to_use = i;
3474
3475         return true;
3476 }
3477
3478 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3479                                    struct sk_buff *skb, u32 tx_flags)
3480 {
3481         struct e1000_adv_tx_context_desc *context_desc;
3482         struct pci_dev *pdev = tx_ring->pdev;
3483         struct igb_buffer *buffer_info;
3484         u32 info = 0, tu_cmd = 0;
3485         unsigned int i;
3486
3487         if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3488             (tx_flags & IGB_TX_FLAGS_VLAN)) {
3489                 i = tx_ring->next_to_use;
3490                 buffer_info = &tx_ring->buffer_info[i];
3491                 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3492
3493                 if (tx_flags & IGB_TX_FLAGS_VLAN)
3494                         info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3495
3496                 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3497                 if (skb->ip_summed == CHECKSUM_PARTIAL)
3498                         info |= skb_network_header_len(skb);
3499
3500                 context_desc->vlan_macip_lens = cpu_to_le32(info);
3501
3502                 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3503
3504                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3505                         __be16 protocol;
3506
3507                         if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3508                                 const struct vlan_ethhdr *vhdr =
3509                                           (const struct vlan_ethhdr*)skb->data;
3510
3511                                 protocol = vhdr->h_vlan_encapsulated_proto;
3512                         } else {
3513                                 protocol = skb->protocol;
3514                         }
3515
3516                         switch (protocol) {
3517                         case cpu_to_be16(ETH_P_IP):
3518                                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3519                                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3520                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3521                                 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3522                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3523                                 break;
3524                         case cpu_to_be16(ETH_P_IPV6):
3525                                 /* XXX what about other V6 headers?? */
3526                                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3527                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3528                                 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3529                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3530                                 break;
3531                         default:
3532                                 if (unlikely(net_ratelimit()))
3533                                         dev_warn(&pdev->dev,
3534                                             "partial checksum but proto=%x!\n",
3535                                             skb->protocol);
3536                                 break;
3537                         }
3538                 }
3539
3540                 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3541                 context_desc->seqnum_seed = 0;
3542                 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3543                         context_desc->mss_l4len_idx =
3544                                 cpu_to_le32(tx_ring->reg_idx << 4);
3545
3546                 buffer_info->time_stamp = jiffies;
3547                 buffer_info->next_to_watch = i;
3548                 buffer_info->dma = 0;
3549
3550                 i++;
3551                 if (i == tx_ring->count)
3552                         i = 0;
3553                 tx_ring->next_to_use = i;
3554
3555                 return true;
3556         }
3557         return false;
3558 }
3559
3560 #define IGB_MAX_TXD_PWR 16
3561 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
3562
3563 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3564                                  unsigned int first)
3565 {
3566         struct igb_buffer *buffer_info;
3567         struct pci_dev *pdev = tx_ring->pdev;
3568         unsigned int len = skb_headlen(skb);
3569         unsigned int count = 0, i;
3570         unsigned int f;
3571
3572         i = tx_ring->next_to_use;
3573
3574         buffer_info = &tx_ring->buffer_info[i];
3575         BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3576         buffer_info->length = len;
3577         /* set time_stamp *before* dma to help avoid a possible race */
3578         buffer_info->time_stamp = jiffies;
3579         buffer_info->next_to_watch = i;
3580         buffer_info->dma = pci_map_single(pdev, skb->data, len,
3581                                           PCI_DMA_TODEVICE);
3582         if (pci_dma_mapping_error(pdev, buffer_info->dma))
3583                 goto dma_error;
3584
3585         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3586                 struct skb_frag_struct *frag;
3587
3588                 count++;
3589                 i++;
3590                 if (i == tx_ring->count)
3591                         i = 0;
3592
3593                 frag = &skb_shinfo(skb)->frags[f];
3594                 len = frag->size;
3595
3596                 buffer_info = &tx_ring->buffer_info[i];
3597                 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3598                 buffer_info->length = len;
3599                 buffer_info->time_stamp = jiffies;
3600                 buffer_info->next_to_watch = i;
3601                 buffer_info->mapped_as_page = true;
3602                 buffer_info->dma = pci_map_page(pdev,
3603                                                 frag->page,
3604                                                 frag->page_offset,
3605                                                 len,
3606                                                 PCI_DMA_TODEVICE);
3607                 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3608                         goto dma_error;
3609
3610         }
3611
3612         tx_ring->buffer_info[i].skb = skb;
3613         tx_ring->buffer_info[first].next_to_watch = i;
3614
3615         return ++count;
3616
3617 dma_error:
3618         dev_err(&pdev->dev, "TX DMA map failed\n");
3619
3620         /* clear timestamp and dma mappings for failed buffer_info mapping */
3621         buffer_info->dma = 0;
3622         buffer_info->time_stamp = 0;
3623         buffer_info->length = 0;
3624         buffer_info->next_to_watch = 0;
3625         buffer_info->mapped_as_page = false;
3626         count--;
3627
3628         /* clear timestamp and dma mappings for remaining portion of packet */
3629         while (count >= 0) {
3630                 count--;
3631                 i--;
3632                 if (i < 0)
3633                         i += tx_ring->count;
3634                 buffer_info = &tx_ring->buffer_info[i];
3635                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3636         }
3637
3638         return 0;
3639 }
3640
3641 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3642                                     int tx_flags, int count, u32 paylen,
3643                                     u8 hdr_len)
3644 {
3645         union e1000_adv_tx_desc *tx_desc;
3646         struct igb_buffer *buffer_info;
3647         u32 olinfo_status = 0, cmd_type_len;
3648         unsigned int i = tx_ring->next_to_use;
3649
3650         cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3651                         E1000_ADVTXD_DCMD_DEXT);
3652
3653         if (tx_flags & IGB_TX_FLAGS_VLAN)
3654                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3655
3656         if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3657                 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3658
3659         if (tx_flags & IGB_TX_FLAGS_TSO) {
3660                 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3661
3662                 /* insert tcp checksum */
3663                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3664
3665                 /* insert ip checksum */
3666                 if (tx_flags & IGB_TX_FLAGS_IPV4)
3667                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3668
3669         } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3670                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3671         }
3672
3673         if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3674             (tx_flags & (IGB_TX_FLAGS_CSUM |
3675                          IGB_TX_FLAGS_TSO |
3676                          IGB_TX_FLAGS_VLAN)))
3677                 olinfo_status |= tx_ring->reg_idx << 4;
3678
3679         olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3680
3681         do {
3682                 buffer_info = &tx_ring->buffer_info[i];
3683                 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3684                 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3685                 tx_desc->read.cmd_type_len =
3686                         cpu_to_le32(cmd_type_len | buffer_info->length);
3687                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3688                 count--;
3689                 i++;
3690                 if (i == tx_ring->count)
3691                         i = 0;
3692         } while (count > 0);
3693
3694         tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
3695         /* Force memory writes to complete before letting h/w
3696          * know there are new descriptors to fetch.  (Only
3697          * applicable for weak-ordered memory model archs,
3698          * such as IA-64). */
3699         wmb();
3700
3701         tx_ring->next_to_use = i;
3702         writel(i, tx_ring->tail);
3703         /* we need this if more than one processor can write to our tail
3704          * at a time, it syncronizes IO on IA64/Altix systems */
3705         mmiowb();
3706 }
3707
3708 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3709 {
3710         struct net_device *netdev = tx_ring->netdev;
3711
3712         netif_stop_subqueue(netdev, tx_ring->queue_index);
3713
3714         /* Herbert's original patch had:
3715          *  smp_mb__after_netif_stop_queue();
3716          * but since that doesn't exist yet, just open code it. */
3717         smp_mb();
3718
3719         /* We need to check again in a case another CPU has just
3720          * made room available. */
3721         if (igb_desc_unused(tx_ring) < size)
3722                 return -EBUSY;
3723
3724         /* A reprieve! */
3725         netif_wake_subqueue(netdev, tx_ring->queue_index);
3726         tx_ring->tx_stats.restart_queue++;
3727         return 0;
3728 }
3729
3730 static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3731 {
3732         if (igb_desc_unused(tx_ring) >= size)
3733                 return 0;
3734         return __igb_maybe_stop_tx(tx_ring, size);
3735 }
3736
3737 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3738                                     struct igb_ring *tx_ring)
3739 {
3740         struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
3741         unsigned int first;
3742         unsigned int tx_flags = 0;
3743         u8 hdr_len = 0;
3744         int tso = 0, count;
3745         union skb_shared_tx *shtx = skb_tx(skb);
3746
3747         /* need: 1 descriptor per page,
3748          *       + 2 desc gap to keep tail from touching head,
3749          *       + 1 desc for skb->data,
3750          *       + 1 desc for context descriptor,
3751          * otherwise try next time */
3752         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
3753                 /* this is a hard error */
3754                 return NETDEV_TX_BUSY;
3755         }
3756
3757         if (unlikely(shtx->hardware)) {
3758                 shtx->in_progress = 1;
3759                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
3760         }
3761
3762         if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
3763                 tx_flags |= IGB_TX_FLAGS_VLAN;
3764                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3765         }
3766
3767         if (skb->protocol == htons(ETH_P_IP))
3768                 tx_flags |= IGB_TX_FLAGS_IPV4;
3769
3770         first = tx_ring->next_to_use;
3771         if (skb_is_gso(skb)) {
3772                 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
3773
3774                 if (tso < 0) {
3775                         dev_kfree_skb_any(skb);
3776                         return NETDEV_TX_OK;
3777                 }
3778         }
3779
3780         if (tso)
3781                 tx_flags |= IGB_TX_FLAGS_TSO;
3782         else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
3783                  (skb->ip_summed == CHECKSUM_PARTIAL))
3784                 tx_flags |= IGB_TX_FLAGS_CSUM;
3785
3786         /*
3787          * count reflects descriptors mapped, if 0 or less then mapping error
3788          * has occured and we need to rewind the descriptor queue
3789          */
3790         count = igb_tx_map_adv(tx_ring, skb, first);
3791         if (!count) {
3792                 dev_kfree_skb_any(skb);
3793                 tx_ring->buffer_info[first].time_stamp = 0;
3794                 tx_ring->next_to_use = first;
3795                 return NETDEV_TX_OK;
3796         }
3797
3798         igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3799
3800         /* Make sure there is space in the ring for the next send. */
3801         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
3802
3803         return NETDEV_TX_OK;
3804 }
3805
3806 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3807                                       struct net_device *netdev)
3808 {
3809         struct igb_adapter *adapter = netdev_priv(netdev);
3810         struct igb_ring *tx_ring;
3811         int r_idx = 0;
3812
3813         if (test_bit(__IGB_DOWN, &adapter->state)) {
3814                 dev_kfree_skb_any(skb);
3815                 return NETDEV_TX_OK;
3816         }
3817
3818         if (skb->len <= 0) {
3819                 dev_kfree_skb_any(skb);
3820                 return NETDEV_TX_OK;
3821         }
3822
3823         r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
3824         tx_ring = adapter->multi_tx_table[r_idx];
3825
3826         /* This goes back to the question of how to logically map a tx queue
3827          * to a flow.  Right now, performance is impacted slightly negatively
3828          * if using multiple tx queues.  If the stack breaks away from a
3829          * single qdisc implementation, we can look at this again. */
3830         return igb_xmit_frame_ring_adv(skb, tx_ring);
3831 }
3832
3833 /**
3834  * igb_tx_timeout - Respond to a Tx Hang
3835  * @netdev: network interface device structure
3836  **/
3837 static void igb_tx_timeout(struct net_device *netdev)
3838 {
3839         struct igb_adapter *adapter = netdev_priv(netdev);
3840         struct e1000_hw *hw = &adapter->hw;
3841
3842         /* Do the reset outside of interrupt context */
3843         adapter->tx_timeout_count++;
3844
3845         if (hw->mac.type == e1000_82580)
3846                 hw->dev_spec._82575.global_device_reset = true;
3847
3848         schedule_work(&adapter->reset_task);
3849         wr32(E1000_EICS,
3850              (adapter->eims_enable_mask & ~adapter->eims_other));
3851 }
3852
3853 static void igb_reset_task(struct work_struct *work)
3854 {
3855         struct igb_adapter *adapter;
3856         adapter = container_of(work, struct igb_adapter, reset_task);
3857
3858         igb_reinit_locked(adapter);
3859 }
3860
3861 /**
3862  * igb_get_stats - Get System Network Statistics
3863  * @netdev: network interface device structure
3864  *
3865  * Returns the address of the device statistics structure.
3866  * The statistics are actually updated from the timer callback.
3867  **/
3868 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
3869 {
3870         /* only return the current stats */
3871         return &netdev->stats;
3872 }
3873
3874 /**
3875  * igb_change_mtu - Change the Maximum Transfer Unit
3876  * @netdev: network interface device structure
3877  * @new_mtu: new value for maximum frame size
3878  *
3879  * Returns 0 on success, negative on failure
3880  **/
3881 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3882 {
3883         struct igb_adapter *adapter = netdev_priv(netdev);
3884         struct pci_dev *pdev = adapter->pdev;
3885         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3886         u32 rx_buffer_len, i;
3887
3888         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3889                 dev_err(&pdev->dev, "Invalid MTU setting\n");
3890                 return -EINVAL;
3891         }
3892
3893         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3894                 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
3895                 return -EINVAL;
3896         }
3897
3898         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3899                 msleep(1);
3900
3901         /* igb_down has a dependency on max_frame_size */
3902         adapter->max_frame_size = max_frame;
3903
3904         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3905          * means we reserve 2 more, this pushes us to allocate from the next
3906          * larger slab size.
3907          * i.e. RXBUFFER_2048 --> size-4096 slab
3908          */
3909
3910         if (max_frame <= IGB_RXBUFFER_1024)
3911                 rx_buffer_len = IGB_RXBUFFER_1024;
3912         else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
3913                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3914         else
3915                 rx_buffer_len = IGB_RXBUFFER_128;
3916
3917         if (netif_running(netdev))
3918                 igb_down(adapter);
3919
3920         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
3921                  netdev->mtu, new_mtu);
3922         netdev->mtu = new_mtu;
3923
3924         for (i = 0; i < adapter->num_rx_queues; i++)
3925                 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3926
3927         if (netif_running(netdev))
3928                 igb_up(adapter);
3929         else
3930                 igb_reset(adapter);
3931
3932         clear_bit(__IGB_RESETTING, &adapter->state);
3933
3934         return 0;
3935 }
3936
3937 /**
3938  * igb_update_stats - Update the board statistics counters
3939  * @adapter: board private structure
3940  **/
3941
3942 void igb_update_stats(struct igb_adapter *adapter)
3943 {
3944         struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
3945         struct e1000_hw *hw = &adapter->hw;
3946         struct pci_dev *pdev = adapter->pdev;
3947         u32 rnbc;
3948         u16 phy_tmp;
3949         int i;
3950         u64 bytes, packets;
3951
3952 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3953
3954         /*
3955          * Prevent stats update while adapter is being reset, or if the pci
3956          * connection is down.
3957          */
3958         if (adapter->link_speed == 0)
3959                 return;
3960         if (pci_channel_offline(pdev))
3961                 return;
3962
3963         bytes = 0;
3964         packets = 0;
3965         for (i = 0; i < adapter->num_rx_queues; i++) {
3966                 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3967                 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3968                 net_stats->rx_fifo_errors += rqdpc_tmp;
3969                 bytes += adapter->rx_ring[i].rx_stats.bytes;
3970                 packets += adapter->rx_ring[i].rx_stats.packets;
3971         }
3972
3973         net_stats->rx_bytes = bytes;
3974         net_stats->rx_packets = packets;
3975
3976         bytes = 0;
3977         packets = 0;
3978         for (i = 0; i < adapter->num_tx_queues; i++) {
3979                 bytes += adapter->tx_ring[i].tx_stats.bytes;
3980                 packets += adapter->tx_ring[i].tx_stats.packets;
3981         }
3982         net_stats->tx_bytes = bytes;
3983         net_stats->tx_packets = packets;
3984
3985         /* read stats registers */
3986         adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3987         adapter->stats.gprc += rd32(E1000_GPRC);
3988         adapter->stats.gorc += rd32(E1000_GORCL);
3989         rd32(E1000_GORCH); /* clear GORCL */
3990         adapter->stats.bprc += rd32(E1000_BPRC);
3991         adapter->stats.mprc += rd32(E1000_MPRC);
3992         adapter->stats.roc += rd32(E1000_ROC);
3993
3994         adapter->stats.prc64 += rd32(E1000_PRC64);
3995         adapter->stats.prc127 += rd32(E1000_PRC127);
3996         adapter->stats.prc255 += rd32(E1000_PRC255);
3997         adapter->stats.prc511 += rd32(E1000_PRC511);
3998         adapter->stats.prc1023 += rd32(E1000_PRC1023);
3999         adapter->stats.prc1522 += rd32(E1000_PRC1522);
4000         adapter->stats.symerrs += rd32(E1000_SYMERRS);
4001         adapter->stats.sec += rd32(E1000_SEC);
4002
4003         adapter->stats.mpc += rd32(E1000_MPC);
4004         adapter->stats.scc += rd32(E1000_SCC);
4005         adapter->stats.ecol += rd32(E1000_ECOL);
4006         adapter->stats.mcc += rd32(E1000_MCC);
4007         adapter->stats.latecol += rd32(E1000_LATECOL);
4008         adapter->stats.dc += rd32(E1000_DC);
4009         adapter->stats.rlec += rd32(E1000_RLEC);
4010         adapter->stats.xonrxc += rd32(E1000_XONRXC);
4011         adapter->stats.xontxc += rd32(E1000_XONTXC);
4012         adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4013         adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4014         adapter->stats.fcruc += rd32(E1000_FCRUC);
4015         adapter->stats.gptc += rd32(E1000_GPTC);
4016         adapter->stats.gotc += rd32(E1000_GOTCL);
4017         rd32(E1000_GOTCH); /* clear GOTCL */
4018         rnbc = rd32(E1000_RNBC);
4019         adapter->stats.rnbc += rnbc;
4020         net_stats->rx_fifo_errors += rnbc;
4021         adapter->stats.ruc += rd32(E1000_RUC);
4022         adapter->stats.rfc += rd32(E1000_RFC);
4023         adapter->stats.rjc += rd32(E1000_RJC);
4024         adapter->stats.tor += rd32(E1000_TORH);
4025         adapter->stats.tot += rd32(E1000_TOTH);
4026         adapter->stats.tpr += rd32(E1000_TPR);
4027
4028         adapter->stats.ptc64 += rd32(E1000_PTC64);
4029         adapter->stats.ptc127 += rd32(E1000_PTC127);
4030         adapter->stats.ptc255 += rd32(E1000_PTC255);
4031         adapter->stats.ptc511 += rd32(E1000_PTC511);
4032         adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4033         adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4034
4035         adapter->stats.mptc += rd32(E1000_MPTC);
4036         adapter->stats.bptc += rd32(E1000_BPTC);
4037
4038         /* used for adaptive IFS */
4039         hw->mac.tx_packet_delta = rd32(E1000_TPT);
4040         adapter->stats.tpt += hw->mac.tx_packet_delta;
4041         hw->mac.collision_delta = rd32(E1000_COLC);
4042         adapter->stats.colc += hw->mac.collision_delta;
4043
4044         adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4045         adapter->stats.rxerrc += rd32(E1000_RXERRC);
4046         adapter->stats.tncrs += rd32(E1000_TNCRS);
4047         adapter->stats.tsctc += rd32(E1000_TSCTC);
4048         adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4049
4050         adapter->stats.iac += rd32(E1000_IAC);
4051         adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4052         adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4053         adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4054         adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4055         adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4056         adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4057         adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4058         adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4059
4060         /* Fill out the OS statistics structure */
4061         net_stats->multicast = adapter->stats.mprc;
4062         net_stats->collisions = adapter->stats.colc;
4063
4064         /* Rx Errors */
4065
4066         /* RLEC on some newer hardware can be incorrect so build
4067          * our own version based on RUC and ROC */
4068         net_stats->rx_errors = adapter->stats.rxerrc +
4069                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4070                 adapter->stats.ruc + adapter->stats.roc +
4071                 adapter->stats.cexterr;
4072         net_stats->rx_length_errors = adapter->stats.ruc +
4073                                       adapter->stats.roc;
4074         net_stats->rx_crc_errors = adapter->stats.crcerrs;
4075         net_stats->rx_frame_errors = adapter->stats.algnerrc;
4076         net_stats->rx_missed_errors = adapter->stats.mpc;
4077
4078         /* Tx Errors */
4079         net_stats->tx_errors = adapter->stats.ecol +
4080                                adapter->stats.latecol;
4081         net_stats->tx_aborted_errors = adapter->stats.ecol;
4082         net_stats->tx_window_errors = adapter->stats.latecol;
4083         net_stats->tx_carrier_errors = adapter->stats.tncrs;
4084
4085         /* Tx Dropped needs to be maintained elsewhere */
4086
4087         /* Phy Stats */
4088         if (hw->phy.media_type == e1000_media_type_copper) {
4089                 if ((adapter->link_speed == SPEED_1000) &&
4090                    (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4091                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4092                         adapter->phy_stats.idle_errors += phy_tmp;
4093                 }
4094         }
4095
4096         /* Management Stats */
4097         adapter->stats.mgptc += rd32(E1000_MGTPTC);
4098         adapter->stats.mgprc += rd32(E1000_MGTPRC);
4099         adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4100 }
4101
4102 static irqreturn_t igb_msix_other(int irq, void *data)
4103 {
4104         struct igb_adapter *adapter = data;
4105         struct e1000_hw *hw = &adapter->hw;
4106         u32 icr = rd32(E1000_ICR);
4107         /* reading ICR causes bit 31 of EICR to be cleared */
4108
4109         if (icr & E1000_ICR_DRSTA)
4110                 schedule_work(&adapter->reset_task);
4111
4112         if (icr & E1000_ICR_DOUTSYNC) {
4113                 /* HW is reporting DMA is out of sync */
4114                 adapter->stats.doosync++;
4115         }
4116
4117         /* Check for a mailbox event */
4118         if (icr & E1000_ICR_VMMB)
4119                 igb_msg_task(adapter);
4120
4121         if (icr & E1000_ICR_LSC) {
4122                 hw->mac.get_link_status = 1;
4123                 /* guard against interrupt when we're going down */
4124                 if (!test_bit(__IGB_DOWN, &adapter->state))
4125                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4126         }
4127
4128         if (adapter->vfs_allocated_count)
4129                 wr32(E1000_IMS, E1000_IMS_LSC |
4130                                 E1000_IMS_VMMB |
4131                                 E1000_IMS_DOUTSYNC);
4132         else
4133                 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4134         wr32(E1000_EIMS, adapter->eims_other);
4135
4136         return IRQ_HANDLED;
4137 }
4138
4139 static void igb_write_itr(struct igb_q_vector *q_vector)
4140 {
4141         u32 itr_val = q_vector->itr_val & 0x7FFC;
4142
4143         if (!q_vector->set_itr)
4144                 return;
4145
4146         if (!itr_val)
4147                 itr_val = 0x4;
4148
4149         if (q_vector->itr_shift)
4150                 itr_val |= itr_val << q_vector->itr_shift;
4151         else
4152                 itr_val |= 0x8000000;
4153
4154         writel(itr_val, q_vector->itr_register);
4155         q_vector->set_itr = 0;
4156 }
4157
4158 static irqreturn_t igb_msix_ring(int irq, void *data)
4159 {
4160         struct igb_q_vector *q_vector = data;
4161
4162         /* Write the ITR value calculated from the previous interrupt. */
4163         igb_write_itr(q_vector);
4164
4165         napi_schedule(&q_vector->napi);
4166
4167         return IRQ_HANDLED;
4168 }
4169
4170 #ifdef CONFIG_IGB_DCA
4171 static void igb_update_dca(struct igb_q_vector *q_vector)
4172 {
4173         struct igb_adapter *adapter = q_vector->adapter;
4174         struct e1000_hw *hw = &adapter->hw;
4175         int cpu = get_cpu();
4176
4177         if (q_vector->cpu == cpu)
4178                 goto out_no_update;
4179
4180         if (q_vector->tx_ring) {
4181                 int q = q_vector->tx_ring->reg_idx;
4182                 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4183                 if (hw->mac.type == e1000_82575) {
4184                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4185                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4186                 } else {
4187                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4188                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4189                                       E1000_DCA_TXCTRL_CPUID_SHIFT;
4190                 }
4191                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4192                 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4193         }
4194         if (q_vector->rx_ring) {
4195                 int q = q_vector->rx_ring->reg_idx;
4196                 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4197                 if (hw->mac.type == e1000_82575) {
4198                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4199                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4200                 } else {
4201                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4202                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4203                                       E1000_DCA_RXCTRL_CPUID_SHIFT;
4204                 }
4205                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4206                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4207                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4208                 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4209         }
4210         q_vector->cpu = cpu;
4211 out_no_update:
4212         put_cpu();
4213 }
4214
4215 static void igb_setup_dca(struct igb_adapter *adapter)
4216 {
4217         struct e1000_hw *hw = &adapter->hw;
4218         int i;
4219
4220         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4221                 return;
4222
4223         /* Always use CB2 mode, difference is masked in the CB driver. */
4224         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4225
4226         for (i = 0; i < adapter->num_q_vectors; i++) {
4227                 struct igb_q_vector *q_vector = adapter->q_vector[i];
4228                 q_vector->cpu = -1;
4229                 igb_update_dca(q_vector);
4230         }
4231 }
4232
4233 static int __igb_notify_dca(struct device *dev, void *data)
4234 {
4235         struct net_device *netdev = dev_get_drvdata(dev);
4236         struct igb_adapter *adapter = netdev_priv(netdev);
4237         struct pci_dev *pdev = adapter->pdev;
4238         struct e1000_hw *hw = &adapter->hw;
4239         unsigned long event = *(unsigned long *)data;
4240
4241         switch (event) {
4242         case DCA_PROVIDER_ADD:
4243                 /* if already enabled, don't do it again */
4244                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4245                         break;
4246                 if (dca_add_requester(dev) == 0) {
4247                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
4248                         dev_info(&pdev->dev, "DCA enabled\n");
4249                         igb_setup_dca(adapter);
4250                         break;
4251                 }
4252                 /* Fall Through since DCA is disabled. */
4253         case DCA_PROVIDER_REMOVE:
4254                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4255                         /* without this a class_device is left
4256                          * hanging around in the sysfs model */
4257                         dca_remove_requester(dev);
4258                         dev_info(&pdev->dev, "DCA disabled\n");
4259                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4260                         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4261                 }
4262                 break;
4263         }
4264
4265         return 0;
4266 }
4267
4268 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4269                           void *p)
4270 {
4271         int ret_val;
4272
4273         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4274                                          __igb_notify_dca);
4275
4276         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4277 }
4278 #endif /* CONFIG_IGB_DCA */
4279
4280 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4281 {
4282         struct e1000_hw *hw = &adapter->hw;
4283         u32 ping;
4284         int i;
4285
4286         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4287                 ping = E1000_PF_CONTROL_MSG;
4288                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4289                         ping |= E1000_VT_MSGTYPE_CTS;
4290                 igb_write_mbx(hw, &ping, 1, i);
4291         }
4292 }
4293
4294 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4295 {
4296         struct e1000_hw *hw = &adapter->hw;
4297         u32 vmolr = rd32(E1000_VMOLR(vf));
4298         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4299
4300         vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4301                             IGB_VF_FLAG_MULTI_PROMISC);
4302         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4303
4304         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4305                 vmolr |= E1000_VMOLR_MPME;
4306                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4307         } else {
4308                 /*
4309                  * if we have hashes and we are clearing a multicast promisc
4310                  * flag we need to write the hashes to the MTA as this step
4311                  * was previously skipped
4312                  */
4313                 if (vf_data->num_vf_mc_hashes > 30) {
4314                         vmolr |= E1000_VMOLR_MPME;
4315                 } else if (vf_data->num_vf_mc_hashes) {
4316                         int j;
4317                         vmolr |= E1000_VMOLR_ROMPE;
4318                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4319                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4320                 }
4321         }
4322
4323         wr32(E1000_VMOLR(vf), vmolr);
4324
4325         /* there are flags left unprocessed, likely not supported */
4326         if (*msgbuf & E1000_VT_MSGINFO_MASK)
4327                 return -EINVAL;
4328
4329         return 0;
4330
4331 }
4332
4333 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4334                                   u32 *msgbuf, u32 vf)
4335 {
4336         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4337         u16 *hash_list = (u16 *)&msgbuf[1];
4338         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4339         int i;
4340
4341         /* salt away the number of multicast addresses assigned
4342          * to this VF for later use to restore when the PF multi cast
4343          * list changes
4344          */
4345         vf_data->num_vf_mc_hashes = n;
4346
4347         /* only up to 30 hash values supported */
4348         if (n > 30)
4349                 n = 30;
4350
4351         /* store the hashes for later use */
4352         for (i = 0; i < n; i++)
4353                 vf_data->vf_mc_hashes[i] = hash_list[i];
4354
4355         /* Flush and reset the mta with the new values */
4356         igb_set_rx_mode(adapter->netdev);
4357
4358         return 0;
4359 }
4360
4361 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4362 {
4363         struct e1000_hw *hw = &adapter->hw;
4364         struct vf_data_storage *vf_data;
4365         int i, j;
4366
4367         for (i = 0; i < adapter->vfs_allocated_count; i++) {
4368                 u32 vmolr = rd32(E1000_VMOLR(i));
4369                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4370
4371                 vf_data = &adapter->vf_data[i];
4372
4373                 if ((vf_data->num_vf_mc_hashes > 30) ||
4374                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4375                         vmolr |= E1000_VMOLR_MPME;
4376                 } else if (vf_data->num_vf_mc_hashes) {
4377                         vmolr |= E1000_VMOLR_ROMPE;
4378                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4379                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4380                 }
4381                 wr32(E1000_VMOLR(i), vmolr);
4382         }
4383 }
4384
4385 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4386 {
4387         struct e1000_hw *hw = &adapter->hw;
4388         u32 pool_mask, reg, vid;
4389         int i;
4390
4391         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4392
4393         /* Find the vlan filter for this id */
4394         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4395                 reg = rd32(E1000_VLVF(i));
4396
4397                 /* remove the vf from the pool */
4398                 reg &= ~pool_mask;
4399
4400                 /* if pool is empty then remove entry from vfta */
4401                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4402                     (reg & E1000_VLVF_VLANID_ENABLE)) {
4403                         reg = 0;
4404                         vid = reg & E1000_VLVF_VLANID_MASK;
4405                         igb_vfta_set(hw, vid, false);
4406                 }
4407
4408                 wr32(E1000_VLVF(i), reg);
4409         }
4410
4411         adapter->vf_data[vf].vlans_enabled = 0;
4412 }
4413
4414 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4415 {
4416         struct e1000_hw *hw = &adapter->hw;
4417         u32 reg, i;
4418
4419         /* The vlvf table only exists on 82576 hardware and newer */
4420         if (hw->mac.type < e1000_82576)
4421                 return -1;
4422
4423         /* we only need to do this if VMDq is enabled */
4424         if (!adapter->vfs_allocated_count)
4425                 return -1;
4426
4427         /* Find the vlan filter for this id */
4428         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4429                 reg = rd32(E1000_VLVF(i));
4430                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4431                     vid == (reg & E1000_VLVF_VLANID_MASK))
4432                         break;
4433         }
4434
4435         if (add) {
4436                 if (i == E1000_VLVF_ARRAY_SIZE) {
4437                         /* Did not find a matching VLAN ID entry that was
4438                          * enabled.  Search for a free filter entry, i.e.
4439                          * one without the enable bit set
4440                          */
4441                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4442                                 reg = rd32(E1000_VLVF(i));
4443                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4444                                         break;
4445                         }
4446                 }
4447                 if (i < E1000_VLVF_ARRAY_SIZE) {
4448                         /* Found an enabled/available entry */
4449                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4450
4451                         /* if !enabled we need to set this up in vfta */
4452                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4453                                 /* add VID to filter table */
4454                                 igb_vfta_set(hw, vid, true);
4455                                 reg |= E1000_VLVF_VLANID_ENABLE;
4456                         }
4457                         reg &= ~E1000_VLVF_VLANID_MASK;
4458                         reg |= vid;
4459                         wr32(E1000_VLVF(i), reg);
4460
4461                         /* do not modify RLPML for PF devices */
4462                         if (vf >= adapter->vfs_allocated_count)
4463                                 return 0;
4464
4465                         if (!adapter->vf_data[vf].vlans_enabled) {
4466                                 u32 size;
4467                                 reg = rd32(E1000_VMOLR(vf));
4468                                 size = reg & E1000_VMOLR_RLPML_MASK;
4469                                 size += 4;
4470                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4471                                 reg |= size;
4472                                 wr32(E1000_VMOLR(vf), reg);
4473                         }
4474
4475                         adapter->vf_data[vf].vlans_enabled++;
4476                         return 0;
4477                 }
4478         } else {
4479                 if (i < E1000_VLVF_ARRAY_SIZE) {
4480                         /* remove vf from the pool */
4481                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4482                         /* if pool is empty then remove entry from vfta */
4483                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4484                                 reg = 0;
4485                                 igb_vfta_set(hw, vid, false);
4486                         }
4487                         wr32(E1000_VLVF(i), reg);
4488
4489                         /* do not modify RLPML for PF devices */
4490                         if (vf >= adapter->vfs_allocated_count)
4491                                 return 0;
4492
4493                         adapter->vf_data[vf].vlans_enabled--;
4494                         if (!adapter->vf_data[vf].vlans_enabled) {
4495                                 u32 size;
4496                                 reg = rd32(E1000_VMOLR(vf));
4497                                 size = reg & E1000_VMOLR_RLPML_MASK;
4498                                 size -= 4;
4499                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4500                                 reg |= size;
4501                                 wr32(E1000_VMOLR(vf), reg);
4502                         }
4503                         return 0;
4504                 }
4505         }
4506         return -1;
4507 }
4508
4509 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4510 {
4511         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4512         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4513
4514         return igb_vlvf_set(adapter, vid, add, vf);
4515 }
4516
4517 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4518 {
4519         /* clear all flags */
4520         adapter->vf_data[vf].flags = 0;
4521         adapter->vf_data[vf].last_nack = jiffies;
4522
4523         /* reset offloads to defaults */
4524         igb_set_vmolr(adapter, vf);
4525
4526         /* reset vlans for device */
4527         igb_clear_vf_vfta(adapter, vf);
4528
4529         /* reset multicast table array for vf */
4530         adapter->vf_data[vf].num_vf_mc_hashes = 0;
4531
4532         /* Flush and reset the mta with the new values */
4533         igb_set_rx_mode(adapter->netdev);
4534 }
4535
4536 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4537 {
4538         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4539
4540         /* generate a new mac address as we were hotplug removed/added */
4541         random_ether_addr(vf_mac);
4542
4543         /* process remaining reset events */
4544         igb_vf_reset(adapter, vf);
4545 }
4546
4547 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4548 {
4549         struct e1000_hw *hw = &adapter->hw;
4550         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4551         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4552         u32 reg, msgbuf[3];
4553         u8 *addr = (u8 *)(&msgbuf[1]);
4554
4555         /* process all the same items cleared in a function level reset */
4556         igb_vf_reset(adapter, vf);
4557
4558         /* set vf mac address */
4559         igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4560
4561         /* enable transmit and receive for vf */
4562         reg = rd32(E1000_VFTE);
4563         wr32(E1000_VFTE, reg | (1 << vf));
4564         reg = rd32(E1000_VFRE);
4565         wr32(E1000_VFRE, reg | (1 << vf));
4566
4567         adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4568
4569         /* reply to reset with ack and vf mac address */
4570         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4571         memcpy(addr, vf_mac, 6);
4572         igb_write_mbx(hw, msgbuf, 3, vf);
4573 }
4574
4575 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4576 {
4577         unsigned char *addr = (char *)&msg[1];
4578         int err = -1;
4579
4580         if (is_valid_ether_addr(addr))
4581                 err = igb_set_vf_mac(adapter, vf, addr);
4582
4583         return err;
4584 }
4585
4586 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4587 {
4588         struct e1000_hw *hw = &adapter->hw;
4589         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4590         u32 msg = E1000_VT_MSGTYPE_NACK;
4591
4592         /* if device isn't clear to send it shouldn't be reading either */
4593         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4594             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4595                 igb_write_mbx(hw, &msg, 1, vf);
4596                 vf_data->last_nack = jiffies;
4597         }
4598 }
4599
4600 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4601 {
4602         struct pci_dev *pdev = adapter->pdev;
4603         u32 msgbuf[E1000_VFMAILBOX_SIZE];
4604         struct e1000_hw *hw = &adapter->hw;
4605         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4606         s32 retval;
4607
4608         retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4609
4610         if (retval) {
4611                 /* if receive failed revoke VF CTS stats and restart init */
4612                 dev_err(&pdev->dev, "Error receiving message from VF\n");
4613                 vf_data->flags &= ~IGB_VF_FLAG_CTS;
4614                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4615                         return;
4616                 goto out;
4617         }
4618
4619         /* this is a message we already processed, do nothing */
4620         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
4621                 return;
4622
4623         /*
4624          * until the vf completes a reset it should not be
4625          * allowed to start any configuration.
4626          */
4627
4628         if (msgbuf[0] == E1000_VF_RESET) {
4629                 igb_vf_reset_msg(adapter, vf);
4630                 return;
4631         }
4632
4633         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4634                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4635                         return;
4636                 retval = -1;
4637                 goto out;
4638         }
4639
4640         switch ((msgbuf[0] & 0xFFFF)) {
4641         case E1000_VF_SET_MAC_ADDR:
4642                 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4643                 break;
4644         case E1000_VF_SET_PROMISC:
4645                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4646                 break;
4647         case E1000_VF_SET_MULTICAST:
4648                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4649                 break;
4650         case E1000_VF_SET_LPE:
4651                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4652                 break;
4653         case E1000_VF_SET_VLAN:
4654                 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4655                 break;
4656         default:
4657                 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4658                 retval = -1;
4659                 break;
4660         }
4661
4662         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4663 out:
4664         /* notify the VF of the results of what it sent us */
4665         if (retval)
4666                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4667         else
4668                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4669
4670         igb_write_mbx(hw, msgbuf, 1, vf);
4671 }
4672
4673 static void igb_msg_task(struct igb_adapter *adapter)
4674 {
4675         struct e1000_hw *hw = &adapter->hw;
4676         u32 vf;
4677
4678         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4679                 /* process any reset requests */
4680                 if (!igb_check_for_rst(hw, vf))
4681                         igb_vf_reset_event(adapter, vf);
4682
4683                 /* process any messages pending */
4684                 if (!igb_check_for_msg(hw, vf))
4685                         igb_rcv_msg_from_vf(adapter, vf);
4686
4687                 /* process any acks */
4688                 if (!igb_check_for_ack(hw, vf))
4689                         igb_rcv_ack_from_vf(adapter, vf);
4690         }
4691 }
4692
4693 /**
4694  *  igb_set_uta - Set unicast filter table address
4695  *  @adapter: board private structure
4696  *
4697  *  The unicast table address is a register array of 32-bit registers.
4698  *  The table is meant to be used in a way similar to how the MTA is used
4699  *  however due to certain limitations in the hardware it is necessary to
4700  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4701  *  enable bit to allow vlan tag stripping when promiscous mode is enabled
4702  **/
4703 static void igb_set_uta(struct igb_adapter *adapter)
4704 {
4705         struct e1000_hw *hw = &adapter->hw;
4706         int i;
4707
4708         /* The UTA table only exists on 82576 hardware and newer */
4709         if (hw->mac.type < e1000_82576)
4710                 return;
4711
4712         /* we only need to do this if VMDq is enabled */
4713         if (!adapter->vfs_allocated_count)
4714                 return;
4715
4716         for (i = 0; i < hw->mac.uta_reg_count; i++)
4717                 array_wr32(E1000_UTA, i, ~0);
4718 }
4719
4720 /**
4721  * igb_intr_msi - Interrupt Handler
4722  * @irq: interrupt number
4723  * @data: pointer to a network interface device structure
4724  **/
4725 static irqreturn_t igb_intr_msi(int irq, void *data)
4726 {
4727         struct igb_adapter *adapter = data;
4728         struct igb_q_vector *q_vector = adapter->q_vector[0];
4729         struct e1000_hw *hw = &adapter->hw;
4730         /* read ICR disables interrupts using IAM */
4731         u32 icr = rd32(E1000_ICR);
4732
4733         igb_write_itr(q_vector);
4734
4735         if (icr & E1000_ICR_DRSTA)
4736                 schedule_work(&adapter->reset_task);
4737
4738         if (icr & E1000_ICR_DOUTSYNC) {
4739                 /* HW is reporting DMA is out of sync */
4740                 adapter->stats.doosync++;
4741         }
4742
4743         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4744                 hw->mac.get_link_status = 1;
4745                 if (!test_bit(__IGB_DOWN, &adapter->state))
4746                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4747         }
4748
4749         napi_schedule(&q_vector->napi);
4750
4751         return IRQ_HANDLED;
4752 }
4753
4754 /**
4755  * igb_intr - Legacy Interrupt Handler
4756  * @irq: interrupt number
4757  * @data: pointer to a network interface device structure
4758  **/
4759 static irqreturn_t igb_intr(int irq, void *data)
4760 {
4761         struct igb_adapter *adapter = data;
4762         struct igb_q_vector *q_vector = adapter->q_vector[0];
4763         struct e1000_hw *hw = &adapter->hw;
4764         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
4765          * need for the IMC write */
4766         u32 icr = rd32(E1000_ICR);
4767         if (!icr)
4768                 return IRQ_NONE;  /* Not our interrupt */
4769
4770         igb_write_itr(q_vector);
4771
4772         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4773          * not set, then the adapter didn't send an interrupt */
4774         if (!(icr & E1000_ICR_INT_ASSERTED))
4775                 return IRQ_NONE;
4776
4777         if (icr & E1000_ICR_DRSTA)
4778                 schedule_work(&adapter->reset_task);
4779
4780         if (icr & E1000_ICR_DOUTSYNC) {
4781                 /* HW is reporting DMA is out of sync */
4782                 adapter->stats.doosync++;
4783         }
4784
4785         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4786                 hw->mac.get_link_status = 1;
4787                 /* guard against interrupt when we're going down */
4788                 if (!test_bit(__IGB_DOWN, &adapter->state))
4789                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4790         }
4791
4792         napi_schedule(&q_vector->napi);
4793
4794         return IRQ_HANDLED;
4795 }
4796
4797 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
4798 {
4799         struct igb_adapter *adapter = q_vector->adapter;
4800         struct e1000_hw *hw = &adapter->hw;
4801
4802         if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4803             (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
4804                 if (!adapter->msix_entries)
4805                         igb_set_itr(adapter);
4806                 else
4807                         igb_update_ring_itr(q_vector);
4808         }
4809
4810         if (!test_bit(__IGB_DOWN, &adapter->state)) {
4811                 if (adapter->msix_entries)
4812                         wr32(E1000_EIMS, q_vector->eims_value);
4813                 else
4814                         igb_irq_enable(adapter);
4815         }
4816 }
4817
4818 /**
4819  * igb_poll - NAPI Rx polling callback
4820  * @napi: napi polling structure
4821  * @budget: count of how many packets we should handle
4822  **/
4823 static int igb_poll(struct napi_struct *napi, int budget)
4824 {
4825         struct igb_q_vector *q_vector = container_of(napi,
4826                                                      struct igb_q_vector,
4827                                                      napi);
4828         int tx_clean_complete = 1, work_done = 0;
4829
4830 #ifdef CONFIG_IGB_DCA
4831         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4832                 igb_update_dca(q_vector);
4833 #endif
4834         if (q_vector->tx_ring)
4835                 tx_clean_complete = igb_clean_tx_irq(q_vector);
4836
4837         if (q_vector->rx_ring)
4838                 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4839
4840         if (!tx_clean_complete)
4841                 work_done = budget;
4842
4843         /* If not enough Rx work done, exit the polling mode */
4844         if (work_done < budget) {
4845                 napi_complete(napi);
4846                 igb_ring_irq_enable(q_vector);
4847         }
4848
4849         return work_done;
4850 }
4851
4852 /**
4853  * igb_systim_to_hwtstamp - convert system time value to hw timestamp
4854  * @adapter: board private structure
4855  * @shhwtstamps: timestamp structure to update
4856  * @regval: unsigned 64bit system time value.
4857  *
4858  * We need to convert the system time value stored in the RX/TXSTMP registers
4859  * into a hwtstamp which can be used by the upper level timestamping functions
4860  */
4861 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4862                                    struct skb_shared_hwtstamps *shhwtstamps,
4863                                    u64 regval)
4864 {
4865         u64 ns;
4866
4867         /*
4868          * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
4869          * 24 to match clock shift we setup earlier.
4870          */
4871         if (adapter->hw.mac.type == e1000_82580)
4872                 regval <<= IGB_82580_TSYNC_SHIFT;
4873
4874         ns = timecounter_cyc2time(&adapter->clock, regval);
4875         timecompare_update(&adapter->compare, ns);
4876         memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4877         shhwtstamps->hwtstamp = ns_to_ktime(ns);
4878         shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4879 }
4880
4881 /**
4882  * igb_tx_hwtstamp - utility function which checks for TX time stamp
4883  * @q_vector: pointer to q_vector containing needed info
4884  * @skb: packet that was just sent
4885  *
4886  * If we were asked to do hardware stamping and such a time stamp is
4887  * available, then it must have been for this skb here because we only
4888  * allow only one such packet into the queue.
4889  */
4890 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
4891 {
4892         struct igb_adapter *adapter = q_vector->adapter;
4893         union skb_shared_tx *shtx = skb_tx(skb);
4894         struct e1000_hw *hw = &adapter->hw;
4895         struct skb_shared_hwtstamps shhwtstamps;
4896         u64 regval;
4897
4898         /* if skb does not support hw timestamp or TX stamp not valid exit */
4899         if (likely(!shtx->hardware) ||
4900             !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4901                 return;
4902
4903         regval = rd32(E1000_TXSTMPL);
4904         regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4905
4906         igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4907         skb_tstamp_tx(skb, &shhwtstamps);
4908 }
4909
4910 /**
4911  * igb_clean_tx_irq - Reclaim resources after transmit completes
4912  * @q_vector: pointer to q_vector containing needed info
4913  * returns true if ring is completely cleaned
4914  **/
4915 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
4916 {
4917         struct igb_adapter *adapter = q_vector->adapter;
4918         struct igb_ring *tx_ring = q_vector->tx_ring;
4919         struct net_device *netdev = tx_ring->netdev;
4920         struct e1000_hw *hw = &adapter->hw;
4921         struct igb_buffer *buffer_info;
4922         struct sk_buff *skb;
4923         union e1000_adv_tx_desc *tx_desc, *eop_desc;
4924         unsigned int total_bytes = 0, total_packets = 0;
4925         unsigned int i, eop, count = 0;
4926         bool cleaned = false;
4927
4928         i = tx_ring->next_to_clean;
4929         eop = tx_ring->buffer_info[i].next_to_watch;
4930         eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4931
4932         while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4933                (count < tx_ring->count)) {
4934                 for (cleaned = false; !cleaned; count++) {
4935                         tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4936                         buffer_info = &tx_ring->buffer_info[i];
4937                         cleaned = (i == eop);
4938                         skb = buffer_info->skb;
4939
4940                         if (skb) {
4941                                 unsigned int segs, bytecount;
4942                                 /* gso_segs is currently only valid for tcp */
4943                                 segs = skb_shinfo(skb)->gso_segs ?: 1;
4944                                 /* multiply data chunks by size of headers */
4945                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
4946                                             skb->len;
4947                                 total_packets += segs;
4948                                 total_bytes += bytecount;
4949
4950                                 igb_tx_hwtstamp(q_vector, skb);
4951                         }
4952
4953                         igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4954                         tx_desc->wb.status = 0;
4955
4956                         i++;
4957                         if (i == tx_ring->count)
4958                                 i = 0;
4959                 }
4960                 eop = tx_ring->buffer_info[i].next_to_watch;
4961                 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4962         }
4963
4964         tx_ring->next_to_clean = i;
4965
4966         if (unlikely(count &&
4967                      netif_carrier_ok(netdev) &&
4968                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
4969                 /* Make sure that anybody stopping the queue after this
4970                  * sees the new next_to_clean.
4971                  */
4972                 smp_mb();
4973                 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4974                     !(test_bit(__IGB_DOWN, &adapter->state))) {
4975                         netif_wake_subqueue(netdev, tx_ring->queue_index);
4976                         tx_ring->tx_stats.restart_queue++;
4977                 }
4978         }
4979
4980         if (tx_ring->detect_tx_hung) {
4981                 /* Detect a transmit hang in hardware, this serializes the
4982                  * check with the clearing of time_stamp and movement of i */
4983                 tx_ring->detect_tx_hung = false;
4984                 if (tx_ring->buffer_info[i].time_stamp &&
4985                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4986                                (adapter->tx_timeout_factor * HZ)) &&
4987                     !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
4988
4989                         /* detected Tx unit hang */
4990                         dev_err(&tx_ring->pdev->dev,
4991                                 "Detected Tx Unit Hang\n"
4992                                 "  Tx Queue             <%d>\n"
4993                                 "  TDH                  <%x>\n"
4994                                 "  TDT                  <%x>\n"
4995                                 "  next_to_use          <%x>\n"
4996                                 "  next_to_clean        <%x>\n"
4997                                 "buffer_info[next_to_clean]\n"
4998                                 "  time_stamp           <%lx>\n"
4999                                 "  next_to_watch        <%x>\n"
5000                                 "  jiffies              <%lx>\n"
5001                                 "  desc.status          <%x>\n",
5002                                 tx_ring->queue_index,
5003                                 readl(tx_ring->head),
5004                                 readl(tx_ring->tail),
5005                                 tx_ring->next_to_use,
5006                                 tx_ring->next_to_clean,
5007                                 tx_ring->buffer_info[eop].time_stamp,
5008                                 eop,
5009                                 jiffies,
5010                                 eop_desc->wb.status);
5011                         netif_stop_subqueue(netdev, tx_ring->queue_index);
5012                 }
5013         }
5014         tx_ring->total_bytes += total_bytes;
5015         tx_ring->total_packets += total_packets;
5016         tx_ring->tx_stats.bytes += total_bytes;
5017         tx_ring->tx_stats.packets += total_packets;
5018         return (count < tx_ring->count);
5019 }
5020
5021 /**
5022  * igb_receive_skb - helper function to handle rx indications
5023  * @q_vector: structure containing interrupt and ring information
5024  * @skb: packet to send up
5025  * @vlan_tag: vlan tag for packet
5026  **/
5027 static void igb_receive_skb(struct igb_q_vector *q_vector,
5028                             struct sk_buff *skb,
5029                             u16 vlan_tag)
5030 {
5031         struct igb_adapter *adapter = q_vector->adapter;
5032
5033         if (vlan_tag)
5034                 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5035                                  vlan_tag, skb);
5036         else
5037                 napi_gro_receive(&q_vector->napi, skb);
5038 }
5039
5040 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5041                                        u32 status_err, struct sk_buff *skb)
5042 {
5043         skb->ip_summed = CHECKSUM_NONE;
5044
5045         /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5046         if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5047              (status_err & E1000_RXD_STAT_IXSM))
5048                 return;
5049
5050         /* TCP/UDP checksum error bit is set */
5051         if (status_err &
5052             (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5053                 /*
5054                  * work around errata with sctp packets where the TCPE aka
5055                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5056                  * packets, (aka let the stack check the crc32c)
5057                  */
5058                 if ((skb->len == 60) &&
5059                     (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
5060                         ring->rx_stats.csum_err++;
5061
5062                 /* let the stack verify checksum errors */
5063                 return;
5064         }
5065         /* It must be a TCP or UDP packet with a valid checksum */
5066         if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5067                 skb->ip_summed = CHECKSUM_UNNECESSARY;
5068
5069         dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
5070 }
5071
5072 static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5073                                    struct sk_buff *skb)
5074 {
5075         struct igb_adapter *adapter = q_vector->adapter;
5076         struct e1000_hw *hw = &adapter->hw;
5077         u64 regval;
5078
5079         /*
5080          * If this bit is set, then the RX registers contain the time stamp. No
5081          * other packet will be time stamped until we read these registers, so
5082          * read the registers to make them available again. Because only one
5083          * packet can be time stamped at a time, we know that the register
5084          * values must belong to this one here and therefore we don't need to
5085          * compare any of the additional attributes stored for it.
5086          *
5087          * If nothing went wrong, then it should have a skb_shared_tx that we
5088          * can turn into a skb_shared_hwtstamps.
5089          */
5090         if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
5091                 return;
5092         if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5093                 return;
5094
5095         regval = rd32(E1000_RXSTMPL);
5096         regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5097
5098         igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5099 }
5100 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5101                                union e1000_adv_rx_desc *rx_desc)
5102 {
5103         /* HW will not DMA in data larger than the given buffer, even if it
5104          * parses the (NFS, of course) header to be larger.  In that case, it
5105          * fills the header buffer and spills the rest into the page.
5106          */
5107         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5108                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5109         if (hlen > rx_ring->rx_buffer_len)
5110                 hlen = rx_ring->rx_buffer_len;
5111         return hlen;
5112 }
5113
5114 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5115                                  int *work_done, int budget)
5116 {
5117         struct igb_ring *rx_ring = q_vector->rx_ring;
5118         struct net_device *netdev = rx_ring->netdev;
5119         struct pci_dev *pdev = rx_ring->pdev;
5120         union e1000_adv_rx_desc *rx_desc , *next_rxd;
5121         struct igb_buffer *buffer_info , *next_buffer;
5122         struct sk_buff *skb;
5123         bool cleaned = false;
5124         int cleaned_count = 0;
5125         int current_node = numa_node_id();
5126         unsigned int total_bytes = 0, total_packets = 0;
5127         unsigned int i;
5128         u32 staterr;
5129         u16 length;
5130         u16 vlan_tag;
5131
5132         i = rx_ring->next_to_clean;
5133         buffer_info = &rx_ring->buffer_info[i];
5134         rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5135         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5136
5137         while (staterr & E1000_RXD_STAT_DD) {
5138                 if (*work_done >= budget)
5139                         break;
5140                 (*work_done)++;
5141
5142                 skb = buffer_info->skb;
5143                 prefetch(skb->data - NET_IP_ALIGN);
5144                 buffer_info->skb = NULL;
5145
5146                 i++;
5147                 if (i == rx_ring->count)
5148                         i = 0;
5149
5150                 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5151                 prefetch(next_rxd);
5152                 next_buffer = &rx_ring->buffer_info[i];
5153
5154                 length = le16_to_cpu(rx_desc->wb.upper.length);
5155                 cleaned = true;
5156                 cleaned_count++;
5157
5158                 if (buffer_info->dma) {
5159                         pci_unmap_single(pdev, buffer_info->dma,
5160                                          rx_ring->rx_buffer_len,
5161                                          PCI_DMA_FROMDEVICE);
5162                         buffer_info->dma = 0;
5163                         if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5164                                 skb_put(skb, length);
5165                                 goto send_up;
5166                         }
5167                         skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5168                 }
5169
5170                 if (length) {
5171                         pci_unmap_page(pdev, buffer_info->page_dma,
5172                                        PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
5173                         buffer_info->page_dma = 0;
5174
5175                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
5176                                                 buffer_info->page,
5177                                                 buffer_info->page_offset,
5178                                                 length);
5179
5180                         if ((page_count(buffer_info->page) != 1) ||
5181                             (page_to_nid(buffer_info->page) != current_node))
5182                                 buffer_info->page = NULL;
5183                         else
5184                                 get_page(buffer_info->page);
5185
5186                         skb->len += length;
5187                         skb->data_len += length;
5188                         skb->truesize += length;
5189                 }
5190
5191                 if (!(staterr & E1000_RXD_STAT_EOP)) {
5192                         buffer_info->skb = next_buffer->skb;
5193                         buffer_info->dma = next_buffer->dma;
5194                         next_buffer->skb = skb;
5195                         next_buffer->dma = 0;
5196                         goto next_desc;
5197                 }
5198 send_up:
5199                 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5200                         dev_kfree_skb_irq(skb);
5201                         goto next_desc;
5202                 }
5203
5204                 igb_rx_hwtstamp(q_vector, staterr, skb);
5205                 total_bytes += skb->len;
5206                 total_packets++;
5207
5208                 igb_rx_checksum_adv(rx_ring, staterr, skb);
5209
5210                 skb->protocol = eth_type_trans(skb, netdev);
5211                 skb_record_rx_queue(skb, rx_ring->queue_index);
5212
5213                 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5214                             le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5215
5216                 igb_receive_skb(q_vector, skb, vlan_tag);
5217
5218 next_desc:
5219                 rx_desc->wb.upper.status_error = 0;
5220
5221                 /* return some buffers to hardware, one at a time is too slow */
5222                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5223                         igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5224                         cleaned_count = 0;
5225                 }
5226
5227                 /* use prefetched values */
5228                 rx_desc = next_rxd;
5229                 buffer_info = next_buffer;
5230                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5231         }
5232
5233         rx_ring->next_to_clean = i;
5234         cleaned_count = igb_desc_unused(rx_ring);
5235
5236         if (cleaned_count)
5237                 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5238
5239         rx_ring->total_packets += total_packets;
5240         rx_ring->total_bytes += total_bytes;
5241         rx_ring->rx_stats.packets += total_packets;
5242         rx_ring->rx_stats.bytes += total_bytes;
5243         return cleaned;
5244 }
5245
5246 /**
5247  * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5248  * @adapter: address of board private structure
5249  **/
5250 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5251 {
5252         struct net_device *netdev = rx_ring->netdev;
5253         union e1000_adv_rx_desc *rx_desc;
5254         struct igb_buffer *buffer_info;
5255         struct sk_buff *skb;
5256         unsigned int i;
5257         int bufsz;
5258
5259         i = rx_ring->next_to_use;
5260         buffer_info = &rx_ring->buffer_info[i];
5261
5262         bufsz = rx_ring->rx_buffer_len;
5263
5264         while (cleaned_count--) {
5265                 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5266
5267                 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5268                         if (!buffer_info->page) {
5269                                 buffer_info->page = netdev_alloc_page(netdev);
5270                                 if (!buffer_info->page) {
5271                                         rx_ring->rx_stats.alloc_failed++;
5272                                         goto no_buffers;
5273                                 }
5274                                 buffer_info->page_offset = 0;
5275                         } else {
5276                                 buffer_info->page_offset ^= PAGE_SIZE / 2;
5277                         }
5278                         buffer_info->page_dma =
5279                                 pci_map_page(rx_ring->pdev, buffer_info->page,
5280                                              buffer_info->page_offset,
5281                                              PAGE_SIZE / 2,
5282                                              PCI_DMA_FROMDEVICE);
5283                         if (pci_dma_mapping_error(rx_ring->pdev,
5284                                                   buffer_info->page_dma)) {
5285                                 buffer_info->page_dma = 0;
5286                                 rx_ring->rx_stats.alloc_failed++;
5287                                 goto no_buffers;
5288                         }
5289                 }
5290
5291                 skb = buffer_info->skb;
5292                 if (!skb) {
5293                         skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5294                         if (!skb) {
5295                                 rx_ring->rx_stats.alloc_failed++;
5296                                 goto no_buffers;
5297                         }
5298
5299                         buffer_info->skb = skb;
5300                 }
5301                 if (!buffer_info->dma) {
5302                         buffer_info->dma = pci_map_single(rx_ring->pdev,
5303                                                           skb->data,
5304                                                           bufsz,
5305                                                           PCI_DMA_FROMDEVICE);
5306                         if (pci_dma_mapping_error(rx_ring->pdev,
5307                                                   buffer_info->dma)) {
5308                                 buffer_info->dma = 0;
5309                                 rx_ring->rx_stats.alloc_failed++;
5310                                 goto no_buffers;
5311                         }
5312                 }
5313                 /* Refresh the desc even if buffer_addrs didn't change because
5314                  * each write-back erases this info. */
5315                 if (bufsz < IGB_RXBUFFER_1024) {
5316                         rx_desc->read.pkt_addr =
5317                              cpu_to_le64(buffer_info->page_dma);
5318                         rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5319                 } else {
5320                         rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5321                         rx_desc->read.hdr_addr = 0;
5322                 }
5323
5324                 i++;
5325                 if (i == rx_ring->count)
5326                         i = 0;
5327                 buffer_info = &rx_ring->buffer_info[i];
5328         }
5329
5330 no_buffers:
5331         if (rx_ring->next_to_use != i) {
5332                 rx_ring->next_to_use = i;
5333                 if (i == 0)
5334                         i = (rx_ring->count - 1);
5335                 else
5336                         i--;
5337
5338                 /* Force memory writes to complete before letting h/w
5339                  * know there are new descriptors to fetch.  (Only
5340                  * applicable for weak-ordered memory model archs,
5341                  * such as IA-64). */
5342                 wmb();
5343                 writel(i, rx_ring->tail);
5344         }
5345 }
5346
5347 /**
5348  * igb_mii_ioctl -
5349  * @netdev:
5350  * @ifreq:
5351  * @cmd:
5352  **/
5353 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5354 {
5355         struct igb_adapter *adapter = netdev_priv(netdev);
5356         struct mii_ioctl_data *data = if_mii(ifr);
5357
5358         if (adapter->hw.phy.media_type != e1000_media_type_copper)
5359                 return -EOPNOTSUPP;
5360
5361         switch (cmd) {
5362         case SIOCGMIIPHY:
5363                 data->phy_id = adapter->hw.phy.addr;
5364                 break;
5365         case SIOCGMIIREG:
5366                 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5367                                      &data->val_out))
5368                         return -EIO;
5369                 break;
5370         case SIOCSMIIREG:
5371         default:
5372                 return -EOPNOTSUPP;
5373         }
5374         return 0;
5375 }
5376
5377 /**
5378  * igb_hwtstamp_ioctl - control hardware time stamping
5379  * @netdev:
5380  * @ifreq:
5381  * @cmd:
5382  *
5383  * Outgoing time stamping can be enabled and disabled. Play nice and
5384  * disable it when requested, although it shouldn't case any overhead
5385  * when no packet needs it. At most one packet in the queue may be
5386  * marked for time stamping, otherwise it would be impossible to tell
5387  * for sure to which packet the hardware time stamp belongs.
5388  *
5389  * Incoming time stamping has to be configured via the hardware
5390  * filters. Not all combinations are supported, in particular event
5391  * type has to be specified. Matching the kind of event packet is
5392  * not supported, with the exception of "all V2 events regardless of
5393  * level 2 or 4".
5394  *
5395  **/
5396 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5397                               struct ifreq *ifr, int cmd)
5398 {
5399         struct igb_adapter *adapter = netdev_priv(netdev);
5400         struct e1000_hw *hw = &adapter->hw;
5401         struct hwtstamp_config config;
5402         u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5403         u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5404         u32 tsync_rx_cfg = 0;
5405         bool is_l4 = false;
5406         bool is_l2 = false;
5407         u32 regval;
5408
5409         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5410                 return -EFAULT;
5411
5412         /* reserved for future extensions */
5413         if (config.flags)
5414                 return -EINVAL;
5415
5416         switch (config.tx_type) {
5417         case HWTSTAMP_TX_OFF:
5418                 tsync_tx_ctl = 0;
5419         case HWTSTAMP_TX_ON:
5420                 break;
5421         default:
5422                 return -ERANGE;
5423         }
5424
5425         switch (config.rx_filter) {
5426         case HWTSTAMP_FILTER_NONE:
5427                 tsync_rx_ctl = 0;
5428                 break;
5429         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5430         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5431         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5432         case HWTSTAMP_FILTER_ALL:
5433                 /*
5434                  * register TSYNCRXCFG must be set, therefore it is not
5435                  * possible to time stamp both Sync and Delay_Req messages
5436                  * => fall back to time stamping all packets
5437                  */
5438                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5439                 config.rx_filter = HWTSTAMP_FILTER_ALL;
5440                 break;
5441         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5442                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5443                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5444                 is_l4 = true;
5445                 break;
5446         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5447                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5448                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5449                 is_l4 = true;
5450                 break;
5451         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5452         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5453                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5454                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5455                 is_l2 = true;
5456                 is_l4 = true;
5457                 config.rx_filter = HWTSTAMP_FILTER_SOME;
5458                 break;
5459         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5460         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5461                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5462                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5463                 is_l2 = true;
5464                 is_l4 = true;
5465                 config.rx_filter = HWTSTAMP_FILTER_SOME;
5466                 break;
5467         case HWTSTAMP_FILTER_PTP_V2_EVENT:
5468         case HWTSTAMP_FILTER_PTP_V2_SYNC:
5469         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5470                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5471                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5472                 is_l2 = true;
5473                 break;
5474         default:
5475                 return -ERANGE;
5476         }
5477
5478         if (hw->mac.type == e1000_82575) {
5479                 if (tsync_rx_ctl | tsync_tx_ctl)
5480                         return -EINVAL;
5481                 return 0;
5482         }
5483
5484         /* enable/disable TX */
5485         regval = rd32(E1000_TSYNCTXCTL);
5486         regval &= ~E1000_TSYNCTXCTL_ENABLED;
5487         regval |= tsync_tx_ctl;
5488         wr32(E1000_TSYNCTXCTL, regval);
5489
5490         /* enable/disable RX */
5491         regval = rd32(E1000_TSYNCRXCTL);
5492         regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5493         regval |= tsync_rx_ctl;
5494         wr32(E1000_TSYNCRXCTL, regval);
5495
5496         /* define which PTP packets are time stamped */
5497         wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5498
5499         /* define ethertype filter for timestamped packets */
5500         if (is_l2)
5501                 wr32(E1000_ETQF(3),
5502                                 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5503                                  E1000_ETQF_1588 | /* enable timestamping */
5504                                  ETH_P_1588));     /* 1588 eth protocol type */
5505         else
5506                 wr32(E1000_ETQF(3), 0);
5507
5508 #define PTP_PORT 319
5509         /* L4 Queue Filter[3]: filter by destination port and protocol */
5510         if (is_l4) {
5511                 u32 ftqf = (IPPROTO_UDP /* UDP */
5512                         | E1000_FTQF_VF_BP /* VF not compared */
5513                         | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5514                         | E1000_FTQF_MASK); /* mask all inputs */
5515                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5516
5517                 wr32(E1000_IMIR(3), htons(PTP_PORT));
5518                 wr32(E1000_IMIREXT(3),
5519                      (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5520                 if (hw->mac.type == e1000_82576) {
5521                         /* enable source port check */
5522                         wr32(E1000_SPQF(3), htons(PTP_PORT));
5523                         ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5524                 }
5525                 wr32(E1000_FTQF(3), ftqf);
5526         } else {
5527                 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5528         }
5529         wrfl();
5530
5531         adapter->hwtstamp_config = config;
5532
5533         /* clear TX/RX time stamp registers, just to be sure */
5534         regval = rd32(E1000_TXSTMPH);
5535         regval = rd32(E1000_RXSTMPH);
5536
5537         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5538                 -EFAULT : 0;
5539 }
5540
5541 /**
5542  * igb_ioctl -
5543  * @netdev:
5544  * @ifreq:
5545  * @cmd:
5546  **/
5547 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5548 {
5549         switch (cmd) {
5550         case SIOCGMIIPHY:
5551         case SIOCGMIIREG:
5552         case SIOCSMIIREG:
5553                 return igb_mii_ioctl(netdev, ifr, cmd);
5554         case SIOCSHWTSTAMP:
5555                 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5556         default:
5557                 return -EOPNOTSUPP;
5558         }
5559 }
5560
5561 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5562 {
5563         struct igb_adapter *adapter = hw->back;
5564         u16 cap_offset;
5565
5566         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5567         if (!cap_offset)
5568                 return -E1000_ERR_CONFIG;
5569
5570         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5571
5572         return 0;
5573 }
5574
5575 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5576 {
5577         struct igb_adapter *adapter = hw->back;
5578         u16 cap_offset;
5579
5580         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5581         if (!cap_offset)
5582                 return -E1000_ERR_CONFIG;
5583
5584         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5585
5586         return 0;
5587 }
5588
5589 static void igb_vlan_rx_register(struct net_device *netdev,
5590                                  struct vlan_group *grp)
5591 {
5592         struct igb_adapter *adapter = netdev_priv(netdev);
5593         struct e1000_hw *hw = &adapter->hw;
5594         u32 ctrl, rctl;
5595
5596         igb_irq_disable(adapter);
5597         adapter->vlgrp = grp;
5598
5599         if (grp) {
5600                 /* enable VLAN tag insert/strip */
5601                 ctrl = rd32(E1000_CTRL);
5602                 ctrl |= E1000_CTRL_VME;
5603                 wr32(E1000_CTRL, ctrl);
5604
5605                 /* Disable CFI check */
5606                 rctl = rd32(E1000_RCTL);
5607                 rctl &= ~E1000_RCTL_CFIEN;
5608                 wr32(E1000_RCTL, rctl);
5609         } else {
5610                 /* disable VLAN tag insert/strip */
5611                 ctrl = rd32(E1000_CTRL);
5612                 ctrl &= ~E1000_CTRL_VME;
5613                 wr32(E1000_CTRL, ctrl);
5614         }
5615
5616         igb_rlpml_set(adapter);
5617
5618         if (!test_bit(__IGB_DOWN, &adapter->state))
5619                 igb_irq_enable(adapter);
5620 }
5621
5622 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5623 {
5624         struct igb_adapter *adapter = netdev_priv(netdev);
5625         struct e1000_hw *hw = &adapter->hw;
5626         int pf_id = adapter->vfs_allocated_count;
5627
5628         /* attempt to add filter to vlvf array */
5629         igb_vlvf_set(adapter, vid, true, pf_id);
5630
5631         /* add the filter since PF can receive vlans w/o entry in vlvf */
5632         igb_vfta_set(hw, vid, true);
5633 }
5634
5635 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5636 {
5637         struct igb_adapter *adapter = netdev_priv(netdev);
5638         struct e1000_hw *hw = &adapter->hw;
5639         int pf_id = adapter->vfs_allocated_count;
5640         s32 err;
5641
5642         igb_irq_disable(adapter);
5643         vlan_group_set_device(adapter->vlgrp, vid, NULL);
5644
5645         if (!test_bit(__IGB_DOWN, &adapter->state))
5646                 igb_irq_enable(adapter);
5647
5648         /* remove vlan from VLVF table array */
5649         err = igb_vlvf_set(adapter, vid, false, pf_id);
5650
5651         /* if vid was not present in VLVF just remove it from table */
5652         if (err)
5653                 igb_vfta_set(hw, vid, false);
5654 }
5655
5656 static void igb_restore_vlan(struct igb_adapter *adapter)
5657 {
5658         igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5659
5660         if (adapter->vlgrp) {
5661                 u16 vid;
5662                 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5663                         if (!vlan_group_get_device(adapter->vlgrp, vid))
5664                                 continue;
5665                         igb_vlan_rx_add_vid(adapter->netdev, vid);
5666                 }
5667         }
5668 }
5669
5670 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5671 {
5672         struct pci_dev *pdev = adapter->pdev;
5673         struct e1000_mac_info *mac = &adapter->hw.mac;
5674
5675         mac->autoneg = 0;
5676
5677         switch (spddplx) {
5678         case SPEED_10 + DUPLEX_HALF:
5679                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5680                 break;
5681         case SPEED_10 + DUPLEX_FULL:
5682                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5683                 break;
5684         case SPEED_100 + DUPLEX_HALF:
5685                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5686                 break;
5687         case SPEED_100 + DUPLEX_FULL:
5688                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5689                 break;
5690         case SPEED_1000 + DUPLEX_FULL:
5691                 mac->autoneg = 1;
5692                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5693                 break;
5694         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5695         default:
5696                 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
5697                 return -EINVAL;
5698         }
5699         return 0;
5700 }
5701
5702 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
5703 {
5704         struct net_device *netdev = pci_get_drvdata(pdev);
5705         struct igb_adapter *adapter = netdev_priv(netdev);
5706         struct e1000_hw *hw = &adapter->hw;
5707         u32 ctrl, rctl, status;
5708         u32 wufc = adapter->wol;
5709 #ifdef CONFIG_PM
5710         int retval = 0;
5711 #endif
5712
5713         netif_device_detach(netdev);
5714
5715         if (netif_running(netdev))
5716                 igb_close(netdev);
5717
5718         igb_clear_interrupt_scheme(adapter);
5719
5720 #ifdef CONFIG_PM
5721         retval = pci_save_state(pdev);
5722         if (retval)
5723                 return retval;
5724 #endif
5725
5726         status = rd32(E1000_STATUS);
5727         if (status & E1000_STATUS_LU)
5728                 wufc &= ~E1000_WUFC_LNKC;
5729
5730         if (wufc) {
5731                 igb_setup_rctl(adapter);
5732                 igb_set_rx_mode(netdev);
5733
5734                 /* turn on all-multi mode if wake on multicast is enabled */
5735                 if (wufc & E1000_WUFC_MC) {
5736                         rctl = rd32(E1000_RCTL);
5737                         rctl |= E1000_RCTL_MPE;
5738                         wr32(E1000_RCTL, rctl);
5739                 }
5740
5741                 ctrl = rd32(E1000_CTRL);
5742                 /* advertise wake from D3Cold */
5743                 #define E1000_CTRL_ADVD3WUC 0x00100000
5744                 /* phy power management enable */
5745                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5746                 ctrl |= E1000_CTRL_ADVD3WUC;
5747                 wr32(E1000_CTRL, ctrl);
5748
5749                 /* Allow time for pending master requests to run */
5750                 igb_disable_pcie_master(hw);
5751
5752                 wr32(E1000_WUC, E1000_WUC_PME_EN);
5753                 wr32(E1000_WUFC, wufc);
5754         } else {
5755                 wr32(E1000_WUC, 0);
5756                 wr32(E1000_WUFC, 0);
5757         }
5758
5759         *enable_wake = wufc || adapter->en_mng_pt;
5760         if (!*enable_wake)
5761                 igb_shutdown_serdes_link_82575(hw);
5762
5763         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
5764          * would have already happened in close and is redundant. */
5765         igb_release_hw_control(adapter);
5766
5767         pci_disable_device(pdev);
5768
5769         return 0;
5770 }
5771
5772 #ifdef CONFIG_PM
5773 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5774 {
5775         int retval;
5776         bool wake;
5777
5778         retval = __igb_shutdown(pdev, &wake);
5779         if (retval)
5780                 return retval;
5781
5782         if (wake) {
5783                 pci_prepare_to_sleep(pdev);
5784         } else {
5785                 pci_wake_from_d3(pdev, false);
5786                 pci_set_power_state(pdev, PCI_D3hot);
5787         }
5788
5789         return 0;
5790 }
5791
5792 static int igb_resume(struct pci_dev *pdev)
5793 {
5794         struct net_device *netdev = pci_get_drvdata(pdev);
5795         struct igb_adapter *adapter = netdev_priv(netdev);
5796         struct e1000_hw *hw = &adapter->hw;
5797         u32 err;
5798
5799         pci_set_power_state(pdev, PCI_D0);
5800         pci_restore_state(pdev);
5801
5802         err = pci_enable_device_mem(pdev);
5803         if (err) {
5804                 dev_err(&pdev->dev,
5805                         "igb: Cannot enable PCI device from suspend\n");
5806                 return err;
5807         }
5808         pci_set_master(pdev);
5809
5810         pci_enable_wake(pdev, PCI_D3hot, 0);
5811         pci_enable_wake(pdev, PCI_D3cold, 0);
5812
5813         if (igb_init_interrupt_scheme(adapter)) {
5814                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5815                 return -ENOMEM;
5816         }
5817
5818         /* e1000_power_up_phy(adapter); */
5819
5820         igb_reset(adapter);
5821
5822         /* let the f/w know that the h/w is now under the control of the
5823          * driver. */
5824         igb_get_hw_control(adapter);
5825
5826         wr32(E1000_WUS, ~0);
5827
5828         if (netif_running(netdev)) {
5829                 err = igb_open(netdev);
5830                 if (err)
5831                         return err;
5832         }
5833
5834         netif_device_attach(netdev);
5835
5836         return 0;
5837 }
5838 #endif
5839
5840 static void igb_shutdown(struct pci_dev *pdev)
5841 {
5842         bool wake;
5843
5844         __igb_shutdown(pdev, &wake);
5845
5846         if (system_state == SYSTEM_POWER_OFF) {
5847                 pci_wake_from_d3(pdev, wake);
5848                 pci_set_power_state(pdev, PCI_D3hot);
5849         }
5850 }
5851
5852 #ifdef CONFIG_NET_POLL_CONTROLLER
5853 /*
5854  * Polling 'interrupt' - used by things like netconsole to send skbs
5855  * without having to re-enable interrupts. It's not called while
5856  * the interrupt routine is executing.
5857  */
5858 static void igb_netpoll(struct net_device *netdev)
5859 {
5860         struct igb_adapter *adapter = netdev_priv(netdev);
5861         struct e1000_hw *hw = &adapter->hw;
5862         int i;
5863
5864         if (!adapter->msix_entries) {
5865                 struct igb_q_vector *q_vector = adapter->q_vector[0];
5866                 igb_irq_disable(adapter);
5867                 napi_schedule(&q_vector->napi);
5868                 return;
5869         }
5870
5871         for (i = 0; i < adapter->num_q_vectors; i++) {
5872                 struct igb_q_vector *q_vector = adapter->q_vector[i];
5873                 wr32(E1000_EIMC, q_vector->eims_value);
5874                 napi_schedule(&q_vector->napi);
5875         }
5876 }
5877 #endif /* CONFIG_NET_POLL_CONTROLLER */
5878
5879 /**
5880  * igb_io_error_detected - called when PCI error is detected
5881  * @pdev: Pointer to PCI device
5882  * @state: The current pci connection state
5883  *
5884  * This function is called after a PCI bus error affecting
5885  * this device has been detected.
5886  */
5887 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5888                                               pci_channel_state_t state)
5889 {
5890         struct net_device *netdev = pci_get_drvdata(pdev);
5891         struct igb_adapter *adapter = netdev_priv(netdev);
5892
5893         netif_device_detach(netdev);
5894
5895         if (state == pci_channel_io_perm_failure)
5896                 return PCI_ERS_RESULT_DISCONNECT;
5897
5898         if (netif_running(netdev))
5899                 igb_down(adapter);
5900         pci_disable_device(pdev);
5901
5902         /* Request a slot slot reset. */
5903         return PCI_ERS_RESULT_NEED_RESET;
5904 }
5905
5906 /**
5907  * igb_io_slot_reset - called after the pci bus has been reset.
5908  * @pdev: Pointer to PCI device
5909  *
5910  * Restart the card from scratch, as if from a cold-boot. Implementation
5911  * resembles the first-half of the igb_resume routine.
5912  */
5913 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5914 {
5915         struct net_device *netdev = pci_get_drvdata(pdev);
5916         struct igb_adapter *adapter = netdev_priv(netdev);
5917         struct e1000_hw *hw = &adapter->hw;
5918         pci_ers_result_t result;
5919         int err;
5920
5921         if (pci_enable_device_mem(pdev)) {
5922                 dev_err(&pdev->dev,
5923                         "Cannot re-enable PCI device after reset.\n");
5924                 result = PCI_ERS_RESULT_DISCONNECT;
5925         } else {
5926                 pci_set_master(pdev);
5927                 pci_restore_state(pdev);
5928
5929                 pci_enable_wake(pdev, PCI_D3hot, 0);
5930                 pci_enable_wake(pdev, PCI_D3cold, 0);
5931
5932                 igb_reset(adapter);
5933                 wr32(E1000_WUS, ~0);
5934                 result = PCI_ERS_RESULT_RECOVERED;
5935         }
5936
5937         err = pci_cleanup_aer_uncorrect_error_status(pdev);
5938         if (err) {
5939                 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5940                         "failed 0x%0x\n", err);
5941                 /* non-fatal, continue */
5942         }
5943
5944         return result;
5945 }
5946
5947 /**
5948  * igb_io_resume - called when traffic can start flowing again.
5949  * @pdev: Pointer to PCI device
5950  *
5951  * This callback is called when the error recovery driver tells us that
5952  * its OK to resume normal operation. Implementation resembles the
5953  * second-half of the igb_resume routine.
5954  */
5955 static void igb_io_resume(struct pci_dev *pdev)
5956 {
5957         struct net_device *netdev = pci_get_drvdata(pdev);
5958         struct igb_adapter *adapter = netdev_priv(netdev);
5959
5960         if (netif_running(netdev)) {
5961                 if (igb_up(adapter)) {
5962                         dev_err(&pdev->dev, "igb_up failed after reset\n");
5963                         return;
5964                 }
5965         }
5966
5967         netif_device_attach(netdev);
5968
5969         /* let the f/w know that the h/w is now under the control of the
5970          * driver. */
5971         igb_get_hw_control(adapter);
5972 }
5973
5974 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5975                              u8 qsel)
5976 {
5977         u32 rar_low, rar_high;
5978         struct e1000_hw *hw = &adapter->hw;
5979
5980         /* HW expects these in little endian so we reverse the byte order
5981          * from network order (big endian) to little endian
5982          */
5983         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5984                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5985         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5986
5987         /* Indicate to hardware the Address is Valid. */
5988         rar_high |= E1000_RAH_AV;
5989
5990         if (hw->mac.type == e1000_82575)
5991                 rar_high |= E1000_RAH_POOL_1 * qsel;
5992         else
5993                 rar_high |= E1000_RAH_POOL_1 << qsel;
5994
5995         wr32(E1000_RAL(index), rar_low);
5996         wrfl();
5997         wr32(E1000_RAH(index), rar_high);
5998         wrfl();
5999 }
6000
6001 static int igb_set_vf_mac(struct igb_adapter *adapter,
6002                           int vf, unsigned char *mac_addr)
6003 {
6004         struct e1000_hw *hw = &adapter->hw;
6005         /* VF MAC addresses start at end of receive addresses and moves
6006          * torwards the first, as a result a collision should not be possible */
6007         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6008
6009         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6010
6011         igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6012
6013         return 0;
6014 }
6015
6016 static void igb_vmm_control(struct igb_adapter *adapter)
6017 {
6018         struct e1000_hw *hw = &adapter->hw;
6019         u32 reg;
6020
6021         /* replication is not supported for 82575 */
6022         if (hw->mac.type == e1000_82575)
6023                 return;
6024
6025         /* enable replication vlan tag stripping */
6026         reg = rd32(E1000_RPLOLR);
6027         reg |= E1000_RPLOLR_STRVLAN;
6028         wr32(E1000_RPLOLR, reg);
6029
6030         /* notify HW that the MAC is adding vlan tags */
6031         reg = rd32(E1000_DTXCTL);
6032         reg |= E1000_DTXCTL_VLAN_ADDED;
6033         wr32(E1000_DTXCTL, reg);
6034
6035         if (adapter->vfs_allocated_count) {
6036                 igb_vmdq_set_loopback_pf(hw, true);
6037                 igb_vmdq_set_replication_pf(hw, true);
6038         } else {
6039                 igb_vmdq_set_loopback_pf(hw, false);
6040                 igb_vmdq_set_replication_pf(hw, false);
6041         }
6042 }
6043
6044 /* igb_main.c */