igb: re-order queues to support cleaner use of ivar on 82576
[safe/jmp/linux-2.6] / drivers / net / igb / igb_main.c
index b64c41a..9331e52 100644 (file)
@@ -42,6 +42,7 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/if_ether.h>
+#include <linux/aer.h>
 #ifdef CONFIG_IGB_DCA
 #include <linux/dca.h>
 #endif
@@ -230,6 +231,40 @@ static void __exit igb_exit_module(void)
 
 module_exit(igb_exit_module);
 
+#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
+/**
+ * igb_cache_ring_register - Descriptor ring to register mapping
+ * @adapter: board private structure to initialize
+ *
+ * Once we know the feature-set enabled for the device, we'll cache
+ * the register offset the descriptor ring is assigned to.
+ **/
+static void igb_cache_ring_register(struct igb_adapter *adapter)
+{
+       int i;
+
+       switch (adapter->hw.mac.type) {
+       case e1000_82576:
+               /* The queues are allocated for virtualization such that VF 0
+                * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
+                * In order to avoid collision we start at the first free queue
+                * and continue consuming queues in the same sequence
+                */
+               for (i = 0; i < adapter->num_rx_queues; i++)
+                       adapter->rx_ring[i].reg_idx = Q_IDX_82576(i);
+               for (i = 0; i < adapter->num_tx_queues; i++)
+                       adapter->tx_ring[i].reg_idx = Q_IDX_82576(i);
+               break;
+       case e1000_82575:
+       default:
+               for (i = 0; i < adapter->num_rx_queues; i++)
+                       adapter->rx_ring[i].reg_idx = i;
+               for (i = 0; i < adapter->num_tx_queues; i++)
+                       adapter->tx_ring[i].reg_idx = i;
+               break;
+       }
+}
+
 /**
  * igb_alloc_queues - Allocate memory for all rings
  * @adapter: board private structure to initialize
@@ -271,6 +306,8 @@ static int igb_alloc_queues(struct igb_adapter *adapter)
                /* set a default napi handler for each rx_ring */
                netif_napi_add(adapter->netdev, &ring->napi, igb_poll, 64);
        }
+
+       igb_cache_ring_register(adapter);
        return 0;
 }
 
@@ -311,36 +348,36 @@ static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue,
                array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
                break;
        case e1000_82576:
-               /* The 82576 uses a table-based method for assigning vectors.
+               /* 82576 uses a table-based method for assigning vectors.
                   Each queue has a single entry in the table to which we write
                   a vector number along with a "valid" bit.  Sadly, the layout
                   of the table is somewhat counterintuitive. */
                if (rx_queue > IGB_N0_QUEUE) {
-                       index = (rx_queue & 0x7);
+                       index = (rx_queue >> 1);
                        ivar = array_rd32(E1000_IVAR0, index);
-                       if (rx_queue < 8) {
-                               /* vector goes into low byte of register */
-                               ivar = ivar & 0xFFFFFF00;
-                               ivar |= msix_vector | E1000_IVAR_VALID;
-                       } else {
+                       if (rx_queue & 0x1) {
                                /* vector goes into third byte of register */
                                ivar = ivar & 0xFF00FFFF;
                                ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
+                       } else {
+                               /* vector goes into low byte of register */
+                               ivar = ivar & 0xFFFFFF00;
+                               ivar |= msix_vector | E1000_IVAR_VALID;
                        }
                        adapter->rx_ring[rx_queue].eims_value= 1 << msix_vector;
                        array_wr32(E1000_IVAR0, index, ivar);
                }
                if (tx_queue > IGB_N0_QUEUE) {
-                       index = (tx_queue & 0x7);
+                       index = (tx_queue >> 1);
                        ivar = array_rd32(E1000_IVAR0, index);
-                       if (tx_queue < 8) {
-                               /* vector goes into second byte of register */
-                               ivar = ivar & 0xFFFF00FF;
-                               ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
-                       } else {
+                       if (tx_queue & 0x1) {
                                /* vector goes into high byte of register */
                                ivar = ivar & 0x00FFFFFF;
                                ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
+                       } else {
+                               /* vector goes into second byte of register */
+                               ivar = ivar & 0xFFFF00FF;
+                               ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
                        }
                        adapter->tx_ring[tx_queue].eims_value= 1 << msix_vector;
                        array_wr32(E1000_IVAR0, index, ivar);
@@ -445,7 +482,7 @@ static int igb_request_msix(struct igb_adapter *adapter)
 
        for (i = 0; i < adapter->num_tx_queues; i++) {
                struct igb_ring *ring = &(adapter->tx_ring[i]);
-               sprintf(ring->name, "%s-tx%d", netdev->name, i);
+               sprintf(ring->name, "%s-tx-%d", netdev->name, i);
                err = request_irq(adapter->msix_entries[vector].vector,
                                  &igb_msix_tx, 0, ring->name,
                                  &(adapter->tx_ring[i]));
@@ -458,7 +495,7 @@ static int igb_request_msix(struct igb_adapter *adapter)
        for (i = 0; i < adapter->num_rx_queues; i++) {
                struct igb_ring *ring = &(adapter->rx_ring[i]);
                if (strlen(netdev->name) < (IFNAMSIZ - 5))
-                       sprintf(ring->name, "%s-rx%d", netdev->name, i);
+                       sprintf(ring->name, "%s-rx-%d", netdev->name, i);
                else
                        memcpy(ring->name, netdev->name, IFNAMSIZ);
                err = request_irq(adapter->msix_entries[vector].vector,
@@ -1049,6 +1086,13 @@ static int __devinit igb_probe(struct pci_dev *pdev,
        if (err)
                goto err_pci_reg;
 
+       err = pci_enable_pcie_error_reporting(pdev);
+       if (err) {
+               dev_err(&pdev->dev, "pci_enable_pcie_error_reporting failed "
+                       "0x%x\n", err);
+               /* non-fatal, continue */
+       }
+
        pci_set_master(pdev);
        pci_save_state(pdev);
 
@@ -1340,6 +1384,7 @@ static void __devexit igb_remove(struct pci_dev *pdev)
 #ifdef CONFIG_IGB_DCA
        struct e1000_hw *hw = &adapter->hw;
 #endif
+       int err;
 
        /* flush_scheduled work may reschedule our watchdog task, so
         * explicitly disable watchdog tasks from being rescheduled  */
@@ -1379,6 +1424,11 @@ static void __devexit igb_remove(struct pci_dev *pdev)
 
        free_netdev(netdev);
 
+       err = pci_disable_pcie_error_reporting(pdev);
+       if (err)
+               dev_err(&pdev->dev,
+                       "pci_disable_pcie_error_reporting failed 0x%x\n", err);
+
        pci_disable_device(pdev);
 }
 
@@ -1561,8 +1611,7 @@ int igb_setup_tx_resources(struct igb_adapter *adapter,
        memset(tx_ring->buffer_info, 0, size);
 
        /* round up to nearest 4K */
-       tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc)
-                       + sizeof(u32);
+       tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
        tx_ring->size = ALIGN(tx_ring->size, 4096);
 
        tx_ring->desc = pci_alloc_consistent(pdev, tx_ring->size,
@@ -1621,43 +1670,37 @@ static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
  **/
 static void igb_configure_tx(struct igb_adapter *adapter)
 {
-       u64 tdba, tdwba;
+       u64 tdba;
        struct e1000_hw *hw = &adapter->hw;
        u32 tctl;
        u32 txdctl, txctrl;
-       int i;
+       int i, j;
 
        for (i = 0; i < adapter->num_tx_queues; i++) {
                struct igb_ring *ring = &(adapter->tx_ring[i]);
-
-               wr32(E1000_TDLEN(i),
+               j = ring->reg_idx;
+               wr32(E1000_TDLEN(j),
                                ring->count * sizeof(struct e1000_tx_desc));
                tdba = ring->dma;
-               wr32(E1000_TDBAL(i),
+               wr32(E1000_TDBAL(j),
                                tdba & 0x00000000ffffffffULL);
-               wr32(E1000_TDBAH(i), tdba >> 32);
+               wr32(E1000_TDBAH(j), tdba >> 32);
 
-               tdwba = ring->dma + ring->count * sizeof(struct e1000_tx_desc);
-               tdwba |= 1; /* enable head wb */
-               wr32(E1000_TDWBAL(i),
-                               tdwba & 0x00000000ffffffffULL);
-               wr32(E1000_TDWBAH(i), tdwba >> 32);
-
-               ring->head = E1000_TDH(i);
-               ring->tail = E1000_TDT(i);
+               ring->head = E1000_TDH(j);
+               ring->tail = E1000_TDT(j);
                writel(0, hw->hw_addr + ring->tail);
                writel(0, hw->hw_addr + ring->head);
-               txdctl = rd32(E1000_TXDCTL(i));
+               txdctl = rd32(E1000_TXDCTL(j));
                txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
-               wr32(E1000_TXDCTL(i), txdctl);
+               wr32(E1000_TXDCTL(j), txdctl);
 
                /* Turn off Relaxed Ordering on head write-backs.  The
                 * writebacks MUST be delivered in order or it will
                 * completely screw up our bookeeping.
                 */
-               txctrl = rd32(E1000_DCA_TXCTRL(i));
+               txctrl = rd32(E1000_DCA_TXCTRL(j));
                txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
-               wr32(E1000_DCA_TXCTRL(i), txctrl);
+               wr32(E1000_DCA_TXCTRL(j), txctrl);
        }
 
 
@@ -1774,14 +1817,14 @@ static void igb_setup_rctl(struct igb_adapter *adapter)
        struct e1000_hw *hw = &adapter->hw;
        u32 rctl;
        u32 srrctl = 0;
-       int i;
+       int i, j;
 
        rctl = rd32(E1000_RCTL);
 
        rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
+       rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
 
-       rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
-               E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
+       rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
                (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
 
        /*
@@ -1791,38 +1834,26 @@ static void igb_setup_rctl(struct igb_adapter *adapter)
        */
        rctl |= E1000_RCTL_SECRC;
 
-       rctl &= ~E1000_RCTL_SBP;
+       /*
+        * disable store bad packets, long packet enable, and clear size bits.
+        */
+       rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_LPE | E1000_RCTL_SZ_256);
 
-       if (adapter->netdev->mtu <= ETH_DATA_LEN)
-               rctl &= ~E1000_RCTL_LPE;
-       else
+       if (adapter->netdev->mtu > ETH_DATA_LEN)
                rctl |= E1000_RCTL_LPE;
-       if (adapter->rx_buffer_len <= IGB_RXBUFFER_2048) {
-               /* Setup buffer sizes */
-               rctl &= ~E1000_RCTL_SZ_4096;
-               rctl |= E1000_RCTL_BSEX;
-               switch (adapter->rx_buffer_len) {
-               case IGB_RXBUFFER_256:
-                       rctl |= E1000_RCTL_SZ_256;
-                       rctl &= ~E1000_RCTL_BSEX;
-                       break;
-               case IGB_RXBUFFER_512:
-                       rctl |= E1000_RCTL_SZ_512;
-                       rctl &= ~E1000_RCTL_BSEX;
-                       break;
-               case IGB_RXBUFFER_1024:
-                       rctl |= E1000_RCTL_SZ_1024;
-                       rctl &= ~E1000_RCTL_BSEX;
-                       break;
-               case IGB_RXBUFFER_2048:
-               default:
-                       rctl |= E1000_RCTL_SZ_2048;
-                       rctl &= ~E1000_RCTL_BSEX;
-                       break;
-               }
-       } else {
-               rctl &= ~E1000_RCTL_BSEX;
-               srrctl = adapter->rx_buffer_len >> E1000_SRRCTL_BSIZEPKT_SHIFT;
+
+       /* Setup buffer sizes */
+       switch (adapter->rx_buffer_len) {
+       case IGB_RXBUFFER_256:
+               rctl |= E1000_RCTL_SZ_256;
+               break;
+       case IGB_RXBUFFER_512:
+               rctl |= E1000_RCTL_SZ_512;
+               break;
+       default:
+               srrctl = ALIGN(adapter->rx_buffer_len, 1024)
+                        >> E1000_SRRCTL_BSIZEPKT_SHIFT;
+               break;
        }
 
        /* 82575 and greater support packet-split where the protocol
@@ -1844,8 +1875,10 @@ static void igb_setup_rctl(struct igb_adapter *adapter)
                srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
        }
 
-       for (i = 0; i < adapter->num_rx_queues; i++)
-               wr32(E1000_SRRCTL(i), srrctl);
+       for (i = 0; i < adapter->num_rx_queues; i++) {
+               j = adapter->rx_ring[i].reg_idx;
+               wr32(E1000_SRRCTL(j), srrctl);
+       }
 
        wr32(E1000_RCTL, rctl);
 }
@@ -1862,7 +1895,7 @@ static void igb_configure_rx(struct igb_adapter *adapter)
        struct e1000_hw *hw = &adapter->hw;
        u32 rctl, rxcsum;
        u32 rxdctl;
-       int i;
+       int i, j;
 
        /* disable receives while setting up the descriptors */
        rctl = rd32(E1000_RCTL);
@@ -1877,25 +1910,26 @@ static void igb_configure_rx(struct igb_adapter *adapter)
         * the Base and Length of the Rx Descriptor Ring */
        for (i = 0; i < adapter->num_rx_queues; i++) {
                struct igb_ring *ring = &(adapter->rx_ring[i]);
+               j = ring->reg_idx;
                rdba = ring->dma;
-               wr32(E1000_RDBAL(i),
+               wr32(E1000_RDBAL(j),
                                rdba & 0x00000000ffffffffULL);
-               wr32(E1000_RDBAH(i), rdba >> 32);
-               wr32(E1000_RDLEN(i),
+               wr32(E1000_RDBAH(j), rdba >> 32);
+               wr32(E1000_RDLEN(j),
                               ring->count * sizeof(union e1000_adv_rx_desc));
 
-               ring->head = E1000_RDH(i);
-               ring->tail = E1000_RDT(i);
+               ring->head = E1000_RDH(j);
+               ring->tail = E1000_RDT(j);
                writel(0, hw->hw_addr + ring->tail);
                writel(0, hw->hw_addr + ring->head);
 
-               rxdctl = rd32(E1000_RXDCTL(i));
+               rxdctl = rd32(E1000_RXDCTL(j));
                rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
                rxdctl &= 0xFFF00000;
                rxdctl |= IGB_RX_PTHRESH;
                rxdctl |= IGB_RX_HTHRESH << 8;
                rxdctl |= IGB_RX_WTHRESH << 16;
-               wr32(E1000_RXDCTL(i), rxdctl);
+               wr32(E1000_RXDCTL(j), rxdctl);
 #ifdef CONFIG_IGB_LRO
                /* Intitial LRO Settings */
                ring->lro_mgr.max_aggr = MAX_LRO_AGGR;
@@ -1925,7 +1959,7 @@ static void igb_configure_rx(struct igb_adapter *adapter)
                        shift = 6;
                for (j = 0; j < (32 * 4); j++) {
                        reta.bytes[j & 3] =
-                               (j % adapter->num_rx_queues) << shift;
+                               adapter->rx_ring[(j % adapter->num_rx_queues)].reg_idx << shift;
                        if ((j & 3) == 3)
                                writel(reta.dword,
                                       hw->hw_addr + E1000_RETA(0) + (j & ~3));
@@ -1983,7 +2017,6 @@ static void igb_configure_rx(struct igb_adapter *adapter)
 
 /**
  * igb_free_tx_resources - Free Tx Resources per Queue
- * @adapter: board private structure
  * @tx_ring: Tx descriptor ring for a specific queue
  *
  * Free all transmit software resources
@@ -2036,7 +2069,6 @@ static void igb_unmap_and_free_tx_resource(struct igb_adapter *adapter,
 
 /**
  * igb_clean_tx_ring - Free Tx Buffers
- * @adapter: board private structure
  * @tx_ring: ring to be cleaned
  **/
 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
@@ -2083,7 +2115,6 @@ static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
 
 /**
  * igb_free_rx_resources - Free Rx Resources
- * @adapter: board private structure
  * @rx_ring: ring to clean the resources from
  *
  * Free all receive software resources
@@ -2123,7 +2154,6 @@ static void igb_free_all_rx_resources(struct igb_adapter *adapter)
 
 /**
  * igb_clean_rx_ring - Free Rx Buffers per Queue
- * @adapter: board private structure
  * @rx_ring: ring to free buffers from
  **/
 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
@@ -2336,9 +2366,10 @@ static void igb_watchdog_task(struct work_struct *work)
                                                   &adapter->link_duplex);
 
                        ctrl = rd32(E1000_CTRL);
-                       dev_info(&adapter->pdev->dev,
-                                "NIC Link is Up %d Mbps %s, "
+                       /* Links status message must follow this format */
+                       printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
                                 "Flow Control: %s\n",
+                                netdev->name,
                                 adapter->link_speed,
                                 adapter->link_duplex == FULL_DUPLEX ?
                                 "Full Duplex" : "Half Duplex",
@@ -2373,7 +2404,9 @@ static void igb_watchdog_task(struct work_struct *work)
                if (netif_carrier_ok(netdev)) {
                        adapter->link_speed = 0;
                        adapter->link_duplex = 0;
-                       dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
+                       /* Links status message must follow this format */
+                       printk(KERN_INFO "igb: %s NIC Link is Down\n",
+                              netdev->name);
                        netif_carrier_off(netdev);
                        netif_tx_stop_all_queues(netdev);
                        if (!test_bit(__IGB_DOWN, &adapter->state))
@@ -2709,6 +2742,7 @@ static inline int igb_tso_adv(struct igb_adapter *adapter,
        context_desc->seqnum_seed = 0;
 
        buffer_info->time_stamp = jiffies;
+       buffer_info->next_to_watch = i;
        buffer_info->dma = 0;
        i++;
        if (i == tx_ring->count)
@@ -2772,6 +2806,7 @@ static inline bool igb_tx_csum_adv(struct igb_adapter *adapter,
                                cpu_to_le32(tx_ring->queue_index << 4);
 
                buffer_info->time_stamp = jiffies;
+               buffer_info->next_to_watch = i;
                buffer_info->dma = 0;
 
                i++;
@@ -2790,8 +2825,8 @@ static inline bool igb_tx_csum_adv(struct igb_adapter *adapter,
 #define IGB_MAX_DATA_PER_TXD   (1<<IGB_MAX_TXD_PWR)
 
 static inline int igb_tx_map_adv(struct igb_adapter *adapter,
-                                struct igb_ring *tx_ring,
-                                struct sk_buff *skb)
+                                struct igb_ring *tx_ring, struct sk_buff *skb,
+                                unsigned int first)
 {
        struct igb_buffer *buffer_info;
        unsigned int len = skb_headlen(skb);
@@ -2805,6 +2840,7 @@ static inline int igb_tx_map_adv(struct igb_adapter *adapter,
        buffer_info->length = len;
        /* set time_stamp *before* dma to help avoid a possible race */
        buffer_info->time_stamp = jiffies;
+       buffer_info->next_to_watch = i;
        buffer_info->dma = pci_map_single(adapter->pdev, skb->data, len,
                                          PCI_DMA_TODEVICE);
        count++;
@@ -2822,6 +2858,7 @@ static inline int igb_tx_map_adv(struct igb_adapter *adapter,
                BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
                buffer_info->length = len;
                buffer_info->time_stamp = jiffies;
+               buffer_info->next_to_watch = i;
                buffer_info->dma = pci_map_page(adapter->pdev,
                                                frag->page,
                                                frag->page_offset,
@@ -2834,8 +2871,9 @@ static inline int igb_tx_map_adv(struct igb_adapter *adapter,
                        i = 0;
        }
 
-       i = (i == 0) ? tx_ring->count - 1 : i - 1;
+       i = ((i == 0) ? tx_ring->count - 1 : i - 1);
        tx_ring->buffer_info[i].skb = skb;
+       tx_ring->buffer_info[first].next_to_watch = i;
 
        return count;
 }
@@ -2942,6 +2980,7 @@ static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
                                   struct igb_ring *tx_ring)
 {
        struct igb_adapter *adapter = netdev_priv(netdev);
+       unsigned int first;
        unsigned int tx_flags = 0;
        unsigned int len;
        u8 hdr_len = 0;
@@ -2978,6 +3017,8 @@ static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
        if (skb->protocol == htons(ETH_P_IP))
                tx_flags |= IGB_TX_FLAGS_IPV4;
 
+       first = tx_ring->next_to_use;
+
        tso = skb_is_gso(skb) ? igb_tso_adv(adapter, tx_ring, skb, tx_flags,
                                              &hdr_len) : 0;
 
@@ -2993,7 +3034,7 @@ static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
                                tx_flags |= IGB_TX_FLAGS_CSUM;
 
        igb_tx_queue_adv(adapter, tx_ring, tx_flags,
-                        igb_tx_map_adv(adapter, tx_ring, skb),
+                        igb_tx_map_adv(adapter, tx_ring, skb, first),
                         skb->len, hdr_len);
 
        netdev->trans_start = jiffies;
@@ -3346,8 +3387,8 @@ static irqreturn_t igb_msix_rx(int irq, void *data)
 
        igb_write_itr(rx_ring);
 
-       if (netif_rx_schedule_prep(adapter->netdev, &rx_ring->napi))
-               __netif_rx_schedule(adapter->netdev, &rx_ring->napi);
+       if (netif_rx_schedule_prep(&rx_ring->napi))
+               __netif_rx_schedule(&rx_ring->napi);
 
 #ifdef CONFIG_IGB_DCA
        if (adapter->flags & IGB_FLAG_DCA_ENABLED)
@@ -3363,7 +3404,7 @@ static void igb_update_rx_dca(struct igb_ring *rx_ring)
        struct igb_adapter *adapter = rx_ring->adapter;
        struct e1000_hw *hw = &adapter->hw;
        int cpu = get_cpu();
-       int q = rx_ring - adapter->rx_ring;
+       int q = rx_ring->reg_idx;
 
        if (rx_ring->cpu != cpu) {
                dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
@@ -3390,7 +3431,7 @@ static void igb_update_tx_dca(struct igb_ring *tx_ring)
        struct igb_adapter *adapter = tx_ring->adapter;
        struct e1000_hw *hw = &adapter->hw;
        int cpu = get_cpu();
-       int q = tx_ring - adapter->tx_ring;
+       int q = tx_ring->reg_idx;
 
        if (tx_ring->cpu != cpu) {
                dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
@@ -3499,7 +3540,7 @@ static irqreturn_t igb_intr_msi(int irq, void *data)
                        mod_timer(&adapter->watchdog_timer, jiffies + 1);
        }
 
-       netif_rx_schedule(netdev, &adapter->rx_ring[0].napi);
+       netif_rx_schedule(&adapter->rx_ring[0].napi);
 
        return IRQ_HANDLED;
 }
@@ -3537,7 +3578,7 @@ static irqreturn_t igb_intr(int irq, void *data)
                        mod_timer(&adapter->watchdog_timer, jiffies + 1);
        }
 
-       netif_rx_schedule(netdev, &adapter->rx_ring[0].napi);
+       netif_rx_schedule(&adapter->rx_ring[0].napi);
 
        return IRQ_HANDLED;
 }
@@ -3572,7 +3613,7 @@ static int igb_poll(struct napi_struct *napi, int budget)
            !netif_running(netdev)) {
                if (adapter->itr_setting & 3)
                        igb_set_itr(adapter);
-               netif_rx_complete(netdev, napi);
+               netif_rx_complete(napi);
                if (!test_bit(__IGB_DOWN, &adapter->state))
                        igb_irq_enable(adapter);
                return 0;
@@ -3598,7 +3639,7 @@ static int igb_clean_rx_ring_msix(struct napi_struct *napi, int budget)
 
        /* If not enough Rx work done, exit the polling mode */
        if ((work_done == 0) || !netif_running(netdev)) {
-               netif_rx_complete(netdev, napi);
+               netif_rx_complete(napi);
 
                if (adapter->itr_setting & 3) {
                        if (adapter->num_rx_queues == 1)
@@ -3616,12 +3657,6 @@ static int igb_clean_rx_ring_msix(struct napi_struct *napi, int budget)
        return 1;
 }
 
-static inline u32 get_head(struct igb_ring *tx_ring)
-{
-       void *end = (struct e1000_tx_desc *)tx_ring->desc + tx_ring->count;
-       return le32_to_cpu(*(volatile __le32 *)end);
-}
-
 /**
  * igb_clean_tx_irq - Reclaim resources after transmit completes
  * @adapter: board private structure
@@ -3630,24 +3665,25 @@ static inline u32 get_head(struct igb_ring *tx_ring)
 static bool igb_clean_tx_irq(struct igb_ring *tx_ring)
 {
        struct igb_adapter *adapter = tx_ring->adapter;
-       struct e1000_hw *hw = &adapter->hw;
        struct net_device *netdev = adapter->netdev;
-       struct e1000_tx_desc *tx_desc;
+       struct e1000_hw *hw = &adapter->hw;
        struct igb_buffer *buffer_info;
        struct sk_buff *skb;
-       unsigned int i;
-       u32 head, oldhead;
-       unsigned int count = 0;
+       union e1000_adv_tx_desc *tx_desc, *eop_desc;
        unsigned int total_bytes = 0, total_packets = 0;
-       bool retval = true;
+       unsigned int i, eop, count = 0;
+       bool cleaned = false;
 
-       rmb();
-       head = get_head(tx_ring);
        i = tx_ring->next_to_clean;
-       while (1) {
-               while (i != head) {
-                       tx_desc = E1000_TX_DESC(*tx_ring, i);
+       eop = tx_ring->buffer_info[i].next_to_watch;
+       eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
+
+       while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
+              (count < tx_ring->count)) {
+               for (cleaned = false; !cleaned; count++) {
+                       tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
                        buffer_info = &tx_ring->buffer_info[i];
+                       cleaned = (i == eop);
                        skb = buffer_info->skb;
 
                        if (skb) {
@@ -3662,25 +3698,17 @@ static bool igb_clean_tx_irq(struct igb_ring *tx_ring)
                        }
 
                        igb_unmap_and_free_tx_resource(adapter, buffer_info);
+                       tx_desc->wb.status = 0;
 
                        i++;
                        if (i == tx_ring->count)
                                i = 0;
-
-                       count++;
-                       if (count == IGB_MAX_TX_CLEAN) {
-                               retval = false;
-                               goto done_cleaning;
-                       }
                }
-               oldhead = head;
-               rmb();
-               head = get_head(tx_ring);
-               if (head == oldhead)
-                       goto done_cleaning;
-       }  /* while (1) */
-
-done_cleaning:
+
+               eop = tx_ring->buffer_info[i].next_to_watch;
+               eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
+       }
+
        tx_ring->next_to_clean = i;
 
        if (unlikely(count &&
@@ -3707,7 +3735,6 @@ done_cleaning:
                    && !(rd32(E1000_STATUS) &
                         E1000_STATUS_TXOFF)) {
 
-                       tx_desc = E1000_TX_DESC(*tx_ring, i);
                        /* detected Tx unit hang */
                        dev_err(&adapter->pdev->dev,
                                "Detected Tx Unit Hang\n"
@@ -3716,9 +3743,9 @@ done_cleaning:
                                "  TDT                  <%x>\n"
                                "  next_to_use          <%x>\n"
                                "  next_to_clean        <%x>\n"
-                               "  head (WB)            <%x>\n"
                                "buffer_info[next_to_clean]\n"
                                "  time_stamp           <%lx>\n"
+                               "  next_to_watch        <%x>\n"
                                "  jiffies              <%lx>\n"
                                "  desc.status          <%x>\n",
                                tx_ring->queue_index,
@@ -3726,10 +3753,10 @@ done_cleaning:
                                readl(adapter->hw.hw_addr + tx_ring->tail),
                                tx_ring->next_to_use,
                                tx_ring->next_to_clean,
-                               head,
                                tx_ring->buffer_info[i].time_stamp,
+                               eop,
                                jiffies,
-                               tx_desc->upper.fields.status);
+                               eop_desc->wb.status);
                        netif_stop_subqueue(netdev, tx_ring->queue_index);
                }
        }
@@ -3739,7 +3766,7 @@ done_cleaning:
        tx_ring->tx_stats.packets += total_packets;
        adapter->net_stats.tx_bytes += total_bytes;
        adapter->net_stats.tx_packets += total_packets;
-       return retval;
+       return (count < tx_ring->count);
 }
 
 #ifdef CONFIG_IGB_LRO
@@ -4479,27 +4506,38 @@ static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
        struct net_device *netdev = pci_get_drvdata(pdev);
        struct igb_adapter *adapter = netdev_priv(netdev);
        struct e1000_hw *hw = &adapter->hw;
+       pci_ers_result_t result;
        int err;
 
        if (adapter->need_ioport)
                err = pci_enable_device(pdev);
        else
                err = pci_enable_device_mem(pdev);
+
        if (err) {
                dev_err(&pdev->dev,
                        "Cannot re-enable PCI device after reset.\n");
-               return PCI_ERS_RESULT_DISCONNECT;
-       }
-       pci_set_master(pdev);
-       pci_restore_state(pdev);
+               result = PCI_ERS_RESULT_DISCONNECT;
+       } else {
+               pci_set_master(pdev);
+               pci_restore_state(pdev);
 
-       pci_enable_wake(pdev, PCI_D3hot, 0);
-       pci_enable_wake(pdev, PCI_D3cold, 0);
+               pci_enable_wake(pdev, PCI_D3hot, 0);
+               pci_enable_wake(pdev, PCI_D3cold, 0);
 
-       igb_reset(adapter);
-       wr32(E1000_WUS, ~0);
+               igb_reset(adapter);
+               wr32(E1000_WUS, ~0);
+               result = PCI_ERS_RESULT_RECOVERED;
+       }
+
+       err = pci_cleanup_aer_uncorrect_error_status(pdev);
+       if (err) {
+               dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
+                       "failed 0x%0x\n", err);
+               /* non-fatal, continue */
+       }
 
-       return PCI_ERS_RESULT_RECOVERED;
+       return result;
 }
 
 /**
@@ -4527,7 +4565,6 @@ static void igb_io_resume(struct pci_dev *pdev)
        /* let the f/w know that the h/w is now under the control of the
         * driver. */
        igb_get_hw_control(adapter);
-
 }
 
 /* igb_main.c */