1 /*******************************************************************************
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2009 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26 *******************************************************************************/
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/ipv6.h>
35 #include <net/checksum.h>
36 #include <net/ip6_checksum.h>
37 #include <linux/net_tstamp.h>
38 #include <linux/mii.h>
39 #include <linux/ethtool.h>
40 #include <linux/if_vlan.h>
41 #include <linux/pci.h>
42 #include <linux/pci-aspm.h>
43 #include <linux/delay.h>
44 #include <linux/interrupt.h>
45 #include <linux/if_ether.h>
46 #include <linux/aer.h>
48 #include <linux/dca.h>
52 #define DRV_VERSION "1.3.16-k2"
53 char igb_driver_name[] = "igb";
54 char igb_driver_version[] = DRV_VERSION;
55 static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
57 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
59 static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
63 static struct pci_device_id igb_pci_tbl[] = {
64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
74 /* required last entry */
78 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
80 void igb_reset(struct igb_adapter *);
81 static int igb_setup_all_tx_resources(struct igb_adapter *);
82 static int igb_setup_all_rx_resources(struct igb_adapter *);
83 static void igb_free_all_tx_resources(struct igb_adapter *);
84 static void igb_free_all_rx_resources(struct igb_adapter *);
85 static void igb_setup_mrqc(struct igb_adapter *);
86 void igb_update_stats(struct igb_adapter *);
87 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
88 static void __devexit igb_remove(struct pci_dev *pdev);
89 static int igb_sw_init(struct igb_adapter *);
90 static int igb_open(struct net_device *);
91 static int igb_close(struct net_device *);
92 static void igb_configure_tx(struct igb_adapter *);
93 static void igb_configure_rx(struct igb_adapter *);
94 static void igb_clean_all_tx_rings(struct igb_adapter *);
95 static void igb_clean_all_rx_rings(struct igb_adapter *);
96 static void igb_clean_tx_ring(struct igb_ring *);
97 static void igb_clean_rx_ring(struct igb_ring *);
98 static void igb_set_rx_mode(struct net_device *);
99 static void igb_update_phy_info(unsigned long);
100 static void igb_watchdog(unsigned long);
101 static void igb_watchdog_task(struct work_struct *);
102 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
103 static struct net_device_stats *igb_get_stats(struct net_device *);
104 static int igb_change_mtu(struct net_device *, int);
105 static int igb_set_mac(struct net_device *, void *);
106 static void igb_set_uta(struct igb_adapter *adapter);
107 static irqreturn_t igb_intr(int irq, void *);
108 static irqreturn_t igb_intr_msi(int irq, void *);
109 static irqreturn_t igb_msix_other(int irq, void *);
110 static irqreturn_t igb_msix_ring(int irq, void *);
111 #ifdef CONFIG_IGB_DCA
112 static void igb_update_dca(struct igb_q_vector *);
113 static void igb_setup_dca(struct igb_adapter *);
114 #endif /* CONFIG_IGB_DCA */
115 static bool igb_clean_tx_irq(struct igb_q_vector *);
116 static int igb_poll(struct napi_struct *, int);
117 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
118 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
119 static void igb_tx_timeout(struct net_device *);
120 static void igb_reset_task(struct work_struct *);
121 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
122 static void igb_vlan_rx_add_vid(struct net_device *, u16);
123 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
124 static void igb_restore_vlan(struct igb_adapter *);
125 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
126 static void igb_ping_all_vfs(struct igb_adapter *);
127 static void igb_msg_task(struct igb_adapter *);
128 static int igb_rcv_msg_from_vf(struct igb_adapter *, u32);
129 static void igb_vmm_control(struct igb_adapter *);
130 static int igb_set_vf_mac(struct igb_adapter *adapter, int, unsigned char *);
131 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
133 static inline void igb_set_vmolr(struct e1000_hw *hw, int vfn)
137 reg_data = rd32(E1000_VMOLR(vfn));
138 reg_data |= E1000_VMOLR_BAM | /* Accept broadcast */
139 E1000_VMOLR_ROMPE | /* Accept packets matched in MTA */
140 E1000_VMOLR_AUPE | /* Accept untagged packets */
141 E1000_VMOLR_STRVLAN; /* Strip vlan tags */
142 wr32(E1000_VMOLR(vfn), reg_data);
145 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
148 struct e1000_hw *hw = &adapter->hw;
151 /* if it isn't the PF check to see if VFs are enabled and
152 * increase the size to support vlan tags */
153 if (vfn < adapter->vfs_allocated_count &&
154 adapter->vf_data[vfn].vlans_enabled)
155 size += VLAN_TAG_SIZE;
157 vmolr = rd32(E1000_VMOLR(vfn));
158 vmolr &= ~E1000_VMOLR_RLPML_MASK;
159 vmolr |= size | E1000_VMOLR_LPE;
160 wr32(E1000_VMOLR(vfn), vmolr);
166 static int igb_suspend(struct pci_dev *, pm_message_t);
167 static int igb_resume(struct pci_dev *);
169 static void igb_shutdown(struct pci_dev *);
170 #ifdef CONFIG_IGB_DCA
171 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
172 static struct notifier_block dca_notifier = {
173 .notifier_call = igb_notify_dca,
178 #ifdef CONFIG_NET_POLL_CONTROLLER
179 /* for netdump / net console */
180 static void igb_netpoll(struct net_device *);
182 #ifdef CONFIG_PCI_IOV
183 static unsigned int max_vfs = 0;
184 module_param(max_vfs, uint, 0);
185 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
186 "per physical function");
187 #endif /* CONFIG_PCI_IOV */
189 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
190 pci_channel_state_t);
191 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
192 static void igb_io_resume(struct pci_dev *);
194 static struct pci_error_handlers igb_err_handler = {
195 .error_detected = igb_io_error_detected,
196 .slot_reset = igb_io_slot_reset,
197 .resume = igb_io_resume,
201 static struct pci_driver igb_driver = {
202 .name = igb_driver_name,
203 .id_table = igb_pci_tbl,
205 .remove = __devexit_p(igb_remove),
207 /* Power Managment Hooks */
208 .suspend = igb_suspend,
209 .resume = igb_resume,
211 .shutdown = igb_shutdown,
212 .err_handler = &igb_err_handler
215 static int global_quad_port_a; /* global quad port a indication */
217 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
218 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
219 MODULE_LICENSE("GPL");
220 MODULE_VERSION(DRV_VERSION);
223 * Scale the NIC clock cycle by a large factor so that
224 * relatively small clock corrections can be added or
225 * substracted at each clock tick. The drawbacks of a
226 * large factor are a) that the clock register overflows
227 * more quickly (not such a big deal) and b) that the
228 * increment per tick has to fit into 24 bits.
231 * TIMINCA = IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS *
233 * TIMINCA += TIMINCA * adjustment [ppm] / 1e9
235 * The base scale factor is intentionally a power of two
236 * so that the division in %struct timecounter can be done with
239 #define IGB_TSYNC_SHIFT (19)
240 #define IGB_TSYNC_SCALE (1<<IGB_TSYNC_SHIFT)
243 * The duration of one clock cycle of the NIC.
245 * @todo This hard-coded value is part of the specification and might change
246 * in future hardware revisions. Add revision check.
248 #define IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS 16
250 #if (IGB_TSYNC_SCALE * IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS) >= (1<<24)
251 # error IGB_TSYNC_SCALE and/or IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS are too large to fit into TIMINCA
255 * igb_read_clock - read raw cycle counter (to be used by time counter)
257 static cycle_t igb_read_clock(const struct cyclecounter *tc)
259 struct igb_adapter *adapter =
260 container_of(tc, struct igb_adapter, cycles);
261 struct e1000_hw *hw = &adapter->hw;
264 stamp = rd32(E1000_SYSTIML);
265 stamp |= (u64)rd32(E1000_SYSTIMH) << 32ULL;
272 * igb_get_hw_dev_name - return device name string
273 * used by hardware layer to print debugging information
275 char *igb_get_hw_dev_name(struct e1000_hw *hw)
277 struct igb_adapter *adapter = hw->back;
278 return adapter->netdev->name;
282 * igb_get_time_str - format current NIC and system time as string
284 static char *igb_get_time_str(struct igb_adapter *adapter,
287 cycle_t hw = adapter->cycles.read(&adapter->cycles);
288 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
290 struct timespec delta;
291 getnstimeofday(&sys);
293 delta = timespec_sub(nic, sys);
296 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
298 (long)nic.tv_sec, nic.tv_nsec,
299 (long)sys.tv_sec, sys.tv_nsec,
300 (long)delta.tv_sec, delta.tv_nsec);
307 * igb_init_module - Driver Registration Routine
309 * igb_init_module is the first routine called when the driver is
310 * loaded. All it does is register with the PCI subsystem.
312 static int __init igb_init_module(void)
315 printk(KERN_INFO "%s - version %s\n",
316 igb_driver_string, igb_driver_version);
318 printk(KERN_INFO "%s\n", igb_copyright);
320 global_quad_port_a = 0;
322 #ifdef CONFIG_IGB_DCA
323 dca_register_notify(&dca_notifier);
326 ret = pci_register_driver(&igb_driver);
330 module_init(igb_init_module);
333 * igb_exit_module - Driver Exit Cleanup Routine
335 * igb_exit_module is called just before the driver is removed
338 static void __exit igb_exit_module(void)
340 #ifdef CONFIG_IGB_DCA
341 dca_unregister_notify(&dca_notifier);
343 pci_unregister_driver(&igb_driver);
346 module_exit(igb_exit_module);
348 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
350 * igb_cache_ring_register - Descriptor ring to register mapping
351 * @adapter: board private structure to initialize
353 * Once we know the feature-set enabled for the device, we'll cache
354 * the register offset the descriptor ring is assigned to.
356 static void igb_cache_ring_register(struct igb_adapter *adapter)
359 u32 rbase_offset = adapter->vfs_allocated_count;
361 switch (adapter->hw.mac.type) {
363 /* The queues are allocated for virtualization such that VF 0
364 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
365 * In order to avoid collision we start at the first free queue
366 * and continue consuming queues in the same sequence
368 for (i = 0; i < adapter->num_rx_queues; i++)
369 adapter->rx_ring[i].reg_idx = rbase_offset +
371 for (i = 0; i < adapter->num_tx_queues; i++)
372 adapter->tx_ring[i].reg_idx = rbase_offset +
377 for (i = 0; i < adapter->num_rx_queues; i++)
378 adapter->rx_ring[i].reg_idx = i;
379 for (i = 0; i < adapter->num_tx_queues; i++)
380 adapter->tx_ring[i].reg_idx = i;
385 static void igb_free_queues(struct igb_adapter *adapter)
387 kfree(adapter->tx_ring);
388 kfree(adapter->rx_ring);
390 adapter->tx_ring = NULL;
391 adapter->rx_ring = NULL;
393 adapter->num_rx_queues = 0;
394 adapter->num_tx_queues = 0;
398 * igb_alloc_queues - Allocate memory for all rings
399 * @adapter: board private structure to initialize
401 * We allocate one ring per queue at run-time since we don't know the
402 * number of queues at compile-time.
404 static int igb_alloc_queues(struct igb_adapter *adapter)
408 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
409 sizeof(struct igb_ring), GFP_KERNEL);
410 if (!adapter->tx_ring)
413 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
414 sizeof(struct igb_ring), GFP_KERNEL);
415 if (!adapter->rx_ring)
418 for (i = 0; i < adapter->num_tx_queues; i++) {
419 struct igb_ring *ring = &(adapter->tx_ring[i]);
420 ring->count = adapter->tx_ring_count;
421 ring->queue_index = i;
422 ring->pdev = adapter->pdev;
423 ring->netdev = adapter->netdev;
424 /* For 82575, context index must be unique per ring. */
425 if (adapter->hw.mac.type == e1000_82575)
426 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
429 for (i = 0; i < adapter->num_rx_queues; i++) {
430 struct igb_ring *ring = &(adapter->rx_ring[i]);
431 ring->count = adapter->rx_ring_count;
432 ring->queue_index = i;
433 ring->pdev = adapter->pdev;
434 ring->netdev = adapter->netdev;
435 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
436 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
437 /* set flag indicating ring supports SCTP checksum offload */
438 if (adapter->hw.mac.type >= e1000_82576)
439 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
442 igb_cache_ring_register(adapter);
447 igb_free_queues(adapter);
452 #define IGB_N0_QUEUE -1
453 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
456 struct igb_adapter *adapter = q_vector->adapter;
457 struct e1000_hw *hw = &adapter->hw;
459 int rx_queue = IGB_N0_QUEUE;
460 int tx_queue = IGB_N0_QUEUE;
462 if (q_vector->rx_ring)
463 rx_queue = q_vector->rx_ring->reg_idx;
464 if (q_vector->tx_ring)
465 tx_queue = q_vector->tx_ring->reg_idx;
467 switch (hw->mac.type) {
469 /* The 82575 assigns vectors using a bitmask, which matches the
470 bitmask for the EICR/EIMS/EIMC registers. To assign one
471 or more queues to a vector, we write the appropriate bits
472 into the MSIXBM register for that vector. */
473 if (rx_queue > IGB_N0_QUEUE)
474 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
475 if (tx_queue > IGB_N0_QUEUE)
476 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
477 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
478 q_vector->eims_value = msixbm;
481 /* 82576 uses a table-based method for assigning vectors.
482 Each queue has a single entry in the table to which we write
483 a vector number along with a "valid" bit. Sadly, the layout
484 of the table is somewhat counterintuitive. */
485 if (rx_queue > IGB_N0_QUEUE) {
486 index = (rx_queue & 0x7);
487 ivar = array_rd32(E1000_IVAR0, index);
489 /* vector goes into low byte of register */
490 ivar = ivar & 0xFFFFFF00;
491 ivar |= msix_vector | E1000_IVAR_VALID;
493 /* vector goes into third byte of register */
494 ivar = ivar & 0xFF00FFFF;
495 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
497 array_wr32(E1000_IVAR0, index, ivar);
499 if (tx_queue > IGB_N0_QUEUE) {
500 index = (tx_queue & 0x7);
501 ivar = array_rd32(E1000_IVAR0, index);
503 /* vector goes into second byte of register */
504 ivar = ivar & 0xFFFF00FF;
505 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
507 /* vector goes into high byte of register */
508 ivar = ivar & 0x00FFFFFF;
509 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
511 array_wr32(E1000_IVAR0, index, ivar);
513 q_vector->eims_value = 1 << msix_vector;
522 * igb_configure_msix - Configure MSI-X hardware
524 * igb_configure_msix sets up the hardware to properly
525 * generate MSI-X interrupts.
527 static void igb_configure_msix(struct igb_adapter *adapter)
531 struct e1000_hw *hw = &adapter->hw;
533 adapter->eims_enable_mask = 0;
535 /* set vector for other causes, i.e. link changes */
536 switch (hw->mac.type) {
538 tmp = rd32(E1000_CTRL_EXT);
539 /* enable MSI-X PBA support*/
540 tmp |= E1000_CTRL_EXT_PBA_CLR;
542 /* Auto-Mask interrupts upon ICR read. */
543 tmp |= E1000_CTRL_EXT_EIAME;
544 tmp |= E1000_CTRL_EXT_IRCA;
546 wr32(E1000_CTRL_EXT, tmp);
548 /* enable msix_other interrupt */
549 array_wr32(E1000_MSIXBM(0), vector++,
551 adapter->eims_other = E1000_EIMS_OTHER;
556 /* Turn on MSI-X capability first, or our settings
557 * won't stick. And it will take days to debug. */
558 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
559 E1000_GPIE_PBA | E1000_GPIE_EIAME |
562 /* enable msix_other interrupt */
563 adapter->eims_other = 1 << vector;
564 tmp = (vector++ | E1000_IVAR_VALID) << 8;
566 wr32(E1000_IVAR_MISC, tmp);
569 /* do nothing, since nothing else supports MSI-X */
571 } /* switch (hw->mac.type) */
573 adapter->eims_enable_mask |= adapter->eims_other;
575 for (i = 0; i < adapter->num_q_vectors; i++) {
576 struct igb_q_vector *q_vector = adapter->q_vector[i];
577 igb_assign_vector(q_vector, vector++);
578 adapter->eims_enable_mask |= q_vector->eims_value;
585 * igb_request_msix - Initialize MSI-X interrupts
587 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
590 static int igb_request_msix(struct igb_adapter *adapter)
592 struct net_device *netdev = adapter->netdev;
593 struct e1000_hw *hw = &adapter->hw;
594 int i, err = 0, vector = 0;
596 err = request_irq(adapter->msix_entries[vector].vector,
597 &igb_msix_other, 0, netdev->name, adapter);
602 for (i = 0; i < adapter->num_q_vectors; i++) {
603 struct igb_q_vector *q_vector = adapter->q_vector[i];
605 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
607 if (q_vector->rx_ring && q_vector->tx_ring)
608 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
609 q_vector->rx_ring->queue_index);
610 else if (q_vector->tx_ring)
611 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
612 q_vector->tx_ring->queue_index);
613 else if (q_vector->rx_ring)
614 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
615 q_vector->rx_ring->queue_index);
617 sprintf(q_vector->name, "%s-unused", netdev->name);
619 err = request_irq(adapter->msix_entries[vector].vector,
620 &igb_msix_ring, 0, q_vector->name,
627 igb_configure_msix(adapter);
633 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
635 if (adapter->msix_entries) {
636 pci_disable_msix(adapter->pdev);
637 kfree(adapter->msix_entries);
638 adapter->msix_entries = NULL;
639 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
640 pci_disable_msi(adapter->pdev);
645 * igb_free_q_vectors - Free memory allocated for interrupt vectors
646 * @adapter: board private structure to initialize
648 * This function frees the memory allocated to the q_vectors. In addition if
649 * NAPI is enabled it will delete any references to the NAPI struct prior
650 * to freeing the q_vector.
652 static void igb_free_q_vectors(struct igb_adapter *adapter)
656 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
657 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
658 adapter->q_vector[v_idx] = NULL;
659 netif_napi_del(&q_vector->napi);
662 adapter->num_q_vectors = 0;
666 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
668 * This function resets the device so that it has 0 rx queues, tx queues, and
669 * MSI-X interrupts allocated.
671 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
673 igb_free_queues(adapter);
674 igb_free_q_vectors(adapter);
675 igb_reset_interrupt_capability(adapter);
679 * igb_set_interrupt_capability - set MSI or MSI-X if supported
681 * Attempt to configure interrupts using the best available
682 * capabilities of the hardware and kernel.
684 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
689 /* Number of supported queues. */
690 adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
691 adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
693 /* start with one vector for every rx queue */
694 numvecs = adapter->num_rx_queues;
696 /* if tx handler is seperate add 1 for every tx queue */
697 numvecs += adapter->num_tx_queues;
699 /* store the number of vectors reserved for queues */
700 adapter->num_q_vectors = numvecs;
702 /* add 1 vector for link status interrupts */
704 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
706 if (!adapter->msix_entries)
709 for (i = 0; i < numvecs; i++)
710 adapter->msix_entries[i].entry = i;
712 err = pci_enable_msix(adapter->pdev,
713 adapter->msix_entries,
718 igb_reset_interrupt_capability(adapter);
720 /* If we can't do MSI-X, try MSI */
722 #ifdef CONFIG_PCI_IOV
723 /* disable SR-IOV for non MSI-X configurations */
724 if (adapter->vf_data) {
725 struct e1000_hw *hw = &adapter->hw;
726 /* disable iov and allow time for transactions to clear */
727 pci_disable_sriov(adapter->pdev);
730 kfree(adapter->vf_data);
731 adapter->vf_data = NULL;
732 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
734 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
737 adapter->num_rx_queues = 1;
738 adapter->num_tx_queues = 1;
739 adapter->num_q_vectors = 1;
740 if (!pci_enable_msi(adapter->pdev))
741 adapter->flags |= IGB_FLAG_HAS_MSI;
743 /* Notify the stack of the (possibly) reduced Tx Queue count. */
744 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
749 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
750 * @adapter: board private structure to initialize
752 * We allocate one q_vector per queue interrupt. If allocation fails we
755 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
757 struct igb_q_vector *q_vector;
758 struct e1000_hw *hw = &adapter->hw;
761 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
762 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
765 q_vector->adapter = adapter;
766 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
767 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
768 q_vector->itr_val = IGB_START_ITR;
769 q_vector->set_itr = 1;
770 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
771 adapter->q_vector[v_idx] = q_vector;
778 q_vector = adapter->q_vector[v_idx];
779 netif_napi_del(&q_vector->napi);
781 adapter->q_vector[v_idx] = NULL;
786 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
787 int ring_idx, int v_idx)
789 struct igb_q_vector *q_vector;
791 q_vector = adapter->q_vector[v_idx];
792 q_vector->rx_ring = &adapter->rx_ring[ring_idx];
793 q_vector->rx_ring->q_vector = q_vector;
794 q_vector->itr_val = adapter->itr;
797 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
798 int ring_idx, int v_idx)
800 struct igb_q_vector *q_vector;
802 q_vector = adapter->q_vector[v_idx];
803 q_vector->tx_ring = &adapter->tx_ring[ring_idx];
804 q_vector->tx_ring->q_vector = q_vector;
805 q_vector->itr_val = adapter->itr;
809 * igb_map_ring_to_vector - maps allocated queues to vectors
811 * This function maps the recently allocated queues to vectors.
813 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
818 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
819 (adapter->num_q_vectors < adapter->num_tx_queues))
822 if (adapter->num_q_vectors >=
823 (adapter->num_rx_queues + adapter->num_tx_queues)) {
824 for (i = 0; i < adapter->num_rx_queues; i++)
825 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
826 for (i = 0; i < adapter->num_tx_queues; i++)
827 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
829 for (i = 0; i < adapter->num_rx_queues; i++) {
830 if (i < adapter->num_tx_queues)
831 igb_map_tx_ring_to_vector(adapter, i, v_idx);
832 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
834 for (; i < adapter->num_tx_queues; i++)
835 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
841 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
843 * This function initializes the interrupts and allocates all of the queues.
845 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
847 struct pci_dev *pdev = adapter->pdev;
850 igb_set_interrupt_capability(adapter);
852 err = igb_alloc_q_vectors(adapter);
854 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
855 goto err_alloc_q_vectors;
858 err = igb_alloc_queues(adapter);
860 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
861 goto err_alloc_queues;
864 err = igb_map_ring_to_vector(adapter);
866 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
873 igb_free_queues(adapter);
875 igb_free_q_vectors(adapter);
877 igb_reset_interrupt_capability(adapter);
882 * igb_request_irq - initialize interrupts
884 * Attempts to configure interrupts using the best available
885 * capabilities of the hardware and kernel.
887 static int igb_request_irq(struct igb_adapter *adapter)
889 struct net_device *netdev = adapter->netdev;
890 struct pci_dev *pdev = adapter->pdev;
891 struct e1000_hw *hw = &adapter->hw;
894 if (adapter->msix_entries) {
895 err = igb_request_msix(adapter);
898 /* fall back to MSI */
899 igb_clear_interrupt_scheme(adapter);
900 if (!pci_enable_msi(adapter->pdev))
901 adapter->flags |= IGB_FLAG_HAS_MSI;
902 igb_free_all_tx_resources(adapter);
903 igb_free_all_rx_resources(adapter);
904 adapter->num_tx_queues = 1;
905 adapter->num_rx_queues = 1;
906 adapter->num_q_vectors = 1;
907 err = igb_alloc_q_vectors(adapter);
910 "Unable to allocate memory for vectors\n");
913 err = igb_alloc_queues(adapter);
916 "Unable to allocate memory for queues\n");
917 igb_free_q_vectors(adapter);
920 igb_setup_all_tx_resources(adapter);
921 igb_setup_all_rx_resources(adapter);
923 switch (hw->mac.type) {
925 wr32(E1000_MSIXBM(0),
926 (E1000_EICR_RX_QUEUE0 |
927 E1000_EICR_TX_QUEUE0 |
931 wr32(E1000_IVAR0, E1000_IVAR_VALID);
938 if (adapter->flags & IGB_FLAG_HAS_MSI) {
939 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
940 netdev->name, adapter);
944 /* fall back to legacy interrupts */
945 igb_reset_interrupt_capability(adapter);
946 adapter->flags &= ~IGB_FLAG_HAS_MSI;
949 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
950 netdev->name, adapter);
953 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
960 static void igb_free_irq(struct igb_adapter *adapter)
962 if (adapter->msix_entries) {
965 free_irq(adapter->msix_entries[vector++].vector, adapter);
967 for (i = 0; i < adapter->num_q_vectors; i++) {
968 struct igb_q_vector *q_vector = adapter->q_vector[i];
969 free_irq(adapter->msix_entries[vector++].vector,
973 free_irq(adapter->pdev->irq, adapter);
978 * igb_irq_disable - Mask off interrupt generation on the NIC
979 * @adapter: board private structure
981 static void igb_irq_disable(struct igb_adapter *adapter)
983 struct e1000_hw *hw = &adapter->hw;
985 if (adapter->msix_entries) {
986 u32 regval = rd32(E1000_EIAM);
987 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
988 wr32(E1000_EIMC, adapter->eims_enable_mask);
989 regval = rd32(E1000_EIAC);
990 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
996 synchronize_irq(adapter->pdev->irq);
1000 * igb_irq_enable - Enable default interrupt generation settings
1001 * @adapter: board private structure
1003 static void igb_irq_enable(struct igb_adapter *adapter)
1005 struct e1000_hw *hw = &adapter->hw;
1007 if (adapter->msix_entries) {
1008 u32 regval = rd32(E1000_EIAC);
1009 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1010 regval = rd32(E1000_EIAM);
1011 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1012 wr32(E1000_EIMS, adapter->eims_enable_mask);
1013 if (adapter->vfs_allocated_count)
1014 wr32(E1000_MBVFIMR, 0xFF);
1015 wr32(E1000_IMS, (E1000_IMS_LSC | E1000_IMS_VMMB |
1016 E1000_IMS_DOUTSYNC));
1018 wr32(E1000_IMS, IMS_ENABLE_MASK);
1019 wr32(E1000_IAM, IMS_ENABLE_MASK);
1023 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1025 struct net_device *netdev = adapter->netdev;
1026 u16 vid = adapter->hw.mng_cookie.vlan_id;
1027 u16 old_vid = adapter->mng_vlan_id;
1028 if (adapter->vlgrp) {
1029 if (!vlan_group_get_device(adapter->vlgrp, vid)) {
1030 if (adapter->hw.mng_cookie.status &
1031 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1032 igb_vlan_rx_add_vid(netdev, vid);
1033 adapter->mng_vlan_id = vid;
1035 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1037 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1039 !vlan_group_get_device(adapter->vlgrp, old_vid))
1040 igb_vlan_rx_kill_vid(netdev, old_vid);
1042 adapter->mng_vlan_id = vid;
1047 * igb_release_hw_control - release control of the h/w to f/w
1048 * @adapter: address of board private structure
1050 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1051 * For ASF and Pass Through versions of f/w this means that the
1052 * driver is no longer loaded.
1055 static void igb_release_hw_control(struct igb_adapter *adapter)
1057 struct e1000_hw *hw = &adapter->hw;
1060 /* Let firmware take over control of h/w */
1061 ctrl_ext = rd32(E1000_CTRL_EXT);
1062 wr32(E1000_CTRL_EXT,
1063 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1068 * igb_get_hw_control - get control of the h/w from f/w
1069 * @adapter: address of board private structure
1071 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1072 * For ASF and Pass Through versions of f/w this means that
1073 * the driver is loaded.
1076 static void igb_get_hw_control(struct igb_adapter *adapter)
1078 struct e1000_hw *hw = &adapter->hw;
1081 /* Let firmware know the driver has taken over */
1082 ctrl_ext = rd32(E1000_CTRL_EXT);
1083 wr32(E1000_CTRL_EXT,
1084 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1088 * igb_configure - configure the hardware for RX and TX
1089 * @adapter: private board structure
1091 static void igb_configure(struct igb_adapter *adapter)
1093 struct net_device *netdev = adapter->netdev;
1096 igb_get_hw_control(adapter);
1097 igb_set_rx_mode(netdev);
1099 igb_restore_vlan(adapter);
1101 igb_setup_tctl(adapter);
1102 igb_setup_mrqc(adapter);
1103 igb_setup_rctl(adapter);
1105 igb_configure_tx(adapter);
1106 igb_configure_rx(adapter);
1108 igb_rx_fifo_flush_82575(&adapter->hw);
1110 /* call igb_desc_unused which always leaves
1111 * at least 1 descriptor unused to make sure
1112 * next_to_use != next_to_clean */
1113 for (i = 0; i < adapter->num_rx_queues; i++) {
1114 struct igb_ring *ring = &adapter->rx_ring[i];
1115 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1119 adapter->tx_queue_len = netdev->tx_queue_len;
1124 * igb_up - Open the interface and prepare it to handle traffic
1125 * @adapter: board private structure
1128 int igb_up(struct igb_adapter *adapter)
1130 struct e1000_hw *hw = &adapter->hw;
1133 /* hardware has been reset, we need to reload some things */
1134 igb_configure(adapter);
1136 clear_bit(__IGB_DOWN, &adapter->state);
1138 for (i = 0; i < adapter->num_q_vectors; i++) {
1139 struct igb_q_vector *q_vector = adapter->q_vector[i];
1140 napi_enable(&q_vector->napi);
1142 if (adapter->msix_entries)
1143 igb_configure_msix(adapter);
1145 igb_set_vmolr(hw, adapter->vfs_allocated_count);
1147 /* Clear any pending interrupts. */
1149 igb_irq_enable(adapter);
1151 /* notify VFs that reset has been completed */
1152 if (adapter->vfs_allocated_count) {
1153 u32 reg_data = rd32(E1000_CTRL_EXT);
1154 reg_data |= E1000_CTRL_EXT_PFRSTD;
1155 wr32(E1000_CTRL_EXT, reg_data);
1158 netif_tx_start_all_queues(adapter->netdev);
1160 /* Fire a link change interrupt to start the watchdog. */
1161 wr32(E1000_ICS, E1000_ICS_LSC);
1165 void igb_down(struct igb_adapter *adapter)
1167 struct e1000_hw *hw = &adapter->hw;
1168 struct net_device *netdev = adapter->netdev;
1172 /* signal that we're down so the interrupt handler does not
1173 * reschedule our watchdog timer */
1174 set_bit(__IGB_DOWN, &adapter->state);
1176 /* disable receives in the hardware */
1177 rctl = rd32(E1000_RCTL);
1178 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1179 /* flush and sleep below */
1181 netif_tx_stop_all_queues(netdev);
1183 /* disable transmits in the hardware */
1184 tctl = rd32(E1000_TCTL);
1185 tctl &= ~E1000_TCTL_EN;
1186 wr32(E1000_TCTL, tctl);
1187 /* flush both disables and wait for them to finish */
1191 for (i = 0; i < adapter->num_q_vectors; i++) {
1192 struct igb_q_vector *q_vector = adapter->q_vector[i];
1193 napi_disable(&q_vector->napi);
1196 igb_irq_disable(adapter);
1198 del_timer_sync(&adapter->watchdog_timer);
1199 del_timer_sync(&adapter->phy_info_timer);
1201 netdev->tx_queue_len = adapter->tx_queue_len;
1202 netif_carrier_off(netdev);
1204 /* record the stats before reset*/
1205 igb_update_stats(adapter);
1207 adapter->link_speed = 0;
1208 adapter->link_duplex = 0;
1210 if (!pci_channel_offline(adapter->pdev))
1212 igb_clean_all_tx_rings(adapter);
1213 igb_clean_all_rx_rings(adapter);
1214 #ifdef CONFIG_IGB_DCA
1216 /* since we reset the hardware DCA settings were cleared */
1217 igb_setup_dca(adapter);
1221 void igb_reinit_locked(struct igb_adapter *adapter)
1223 WARN_ON(in_interrupt());
1224 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1228 clear_bit(__IGB_RESETTING, &adapter->state);
1231 void igb_reset(struct igb_adapter *adapter)
1233 struct e1000_hw *hw = &adapter->hw;
1234 struct e1000_mac_info *mac = &hw->mac;
1235 struct e1000_fc_info *fc = &hw->fc;
1236 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1239 /* Repartition Pba for greater than 9k mtu
1240 * To take effect CTRL.RST is required.
1242 switch (mac->type) {
1244 pba = E1000_PBA_64K;
1248 pba = E1000_PBA_34K;
1252 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1253 (mac->type < e1000_82576)) {
1254 /* adjust PBA for jumbo frames */
1255 wr32(E1000_PBA, pba);
1257 /* To maintain wire speed transmits, the Tx FIFO should be
1258 * large enough to accommodate two full transmit packets,
1259 * rounded up to the next 1KB and expressed in KB. Likewise,
1260 * the Rx FIFO should be large enough to accommodate at least
1261 * one full receive packet and is similarly rounded up and
1262 * expressed in KB. */
1263 pba = rd32(E1000_PBA);
1264 /* upper 16 bits has Tx packet buffer allocation size in KB */
1265 tx_space = pba >> 16;
1266 /* lower 16 bits has Rx packet buffer allocation size in KB */
1268 /* the tx fifo also stores 16 bytes of information about the tx
1269 * but don't include ethernet FCS because hardware appends it */
1270 min_tx_space = (adapter->max_frame_size +
1271 sizeof(union e1000_adv_tx_desc) -
1273 min_tx_space = ALIGN(min_tx_space, 1024);
1274 min_tx_space >>= 10;
1275 /* software strips receive CRC, so leave room for it */
1276 min_rx_space = adapter->max_frame_size;
1277 min_rx_space = ALIGN(min_rx_space, 1024);
1278 min_rx_space >>= 10;
1280 /* If current Tx allocation is less than the min Tx FIFO size,
1281 * and the min Tx FIFO size is less than the current Rx FIFO
1282 * allocation, take space away from current Rx allocation */
1283 if (tx_space < min_tx_space &&
1284 ((min_tx_space - tx_space) < pba)) {
1285 pba = pba - (min_tx_space - tx_space);
1287 /* if short on rx space, rx wins and must trump tx
1289 if (pba < min_rx_space)
1292 wr32(E1000_PBA, pba);
1295 /* flow control settings */
1296 /* The high water mark must be low enough to fit one full frame
1297 * (or the size used for early receive) above it in the Rx FIFO.
1298 * Set it to the lower of:
1299 * - 90% of the Rx FIFO size, or
1300 * - the full Rx FIFO size minus one full frame */
1301 hwm = min(((pba << 10) * 9 / 10),
1302 ((pba << 10) - 2 * adapter->max_frame_size));
1304 if (mac->type < e1000_82576) {
1305 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
1306 fc->low_water = fc->high_water - 8;
1308 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1309 fc->low_water = fc->high_water - 16;
1311 fc->pause_time = 0xFFFF;
1313 fc->current_mode = fc->requested_mode;
1315 /* disable receive for all VFs and wait one second */
1316 if (adapter->vfs_allocated_count) {
1318 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1319 adapter->vf_data[i].clear_to_send = false;
1321 /* ping all the active vfs to let them know we are going down */
1322 igb_ping_all_vfs(adapter);
1324 /* disable transmits and receives */
1325 wr32(E1000_VFRE, 0);
1326 wr32(E1000_VFTE, 0);
1329 /* Allow time for pending master requests to run */
1330 adapter->hw.mac.ops.reset_hw(&adapter->hw);
1333 if (adapter->hw.mac.ops.init_hw(&adapter->hw))
1334 dev_err(&adapter->pdev->dev, "Hardware Error\n");
1336 igb_update_mng_vlan(adapter);
1338 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1339 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1341 igb_reset_adaptive(&adapter->hw);
1342 igb_get_phy_info(&adapter->hw);
1345 static const struct net_device_ops igb_netdev_ops = {
1346 .ndo_open = igb_open,
1347 .ndo_stop = igb_close,
1348 .ndo_start_xmit = igb_xmit_frame_adv,
1349 .ndo_get_stats = igb_get_stats,
1350 .ndo_set_rx_mode = igb_set_rx_mode,
1351 .ndo_set_multicast_list = igb_set_rx_mode,
1352 .ndo_set_mac_address = igb_set_mac,
1353 .ndo_change_mtu = igb_change_mtu,
1354 .ndo_do_ioctl = igb_ioctl,
1355 .ndo_tx_timeout = igb_tx_timeout,
1356 .ndo_validate_addr = eth_validate_addr,
1357 .ndo_vlan_rx_register = igb_vlan_rx_register,
1358 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1359 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1360 #ifdef CONFIG_NET_POLL_CONTROLLER
1361 .ndo_poll_controller = igb_netpoll,
1366 * igb_probe - Device Initialization Routine
1367 * @pdev: PCI device information struct
1368 * @ent: entry in igb_pci_tbl
1370 * Returns 0 on success, negative on failure
1372 * igb_probe initializes an adapter identified by a pci_dev structure.
1373 * The OS initialization, configuring of the adapter private structure,
1374 * and a hardware reset occur.
1376 static int __devinit igb_probe(struct pci_dev *pdev,
1377 const struct pci_device_id *ent)
1379 struct net_device *netdev;
1380 struct igb_adapter *adapter;
1381 struct e1000_hw *hw;
1382 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1383 unsigned long mmio_start, mmio_len;
1384 int err, pci_using_dac;
1385 u16 eeprom_data = 0;
1386 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1389 err = pci_enable_device_mem(pdev);
1394 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1396 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1400 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1402 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1404 dev_err(&pdev->dev, "No usable DMA "
1405 "configuration, aborting\n");
1411 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1417 pci_enable_pcie_error_reporting(pdev);
1419 pci_set_master(pdev);
1420 pci_save_state(pdev);
1423 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1424 IGB_ABS_MAX_TX_QUEUES);
1426 goto err_alloc_etherdev;
1428 SET_NETDEV_DEV(netdev, &pdev->dev);
1430 pci_set_drvdata(pdev, netdev);
1431 adapter = netdev_priv(netdev);
1432 adapter->netdev = netdev;
1433 adapter->pdev = pdev;
1436 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1438 mmio_start = pci_resource_start(pdev, 0);
1439 mmio_len = pci_resource_len(pdev, 0);
1442 hw->hw_addr = ioremap(mmio_start, mmio_len);
1446 netdev->netdev_ops = &igb_netdev_ops;
1447 igb_set_ethtool_ops(netdev);
1448 netdev->watchdog_timeo = 5 * HZ;
1450 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1452 netdev->mem_start = mmio_start;
1453 netdev->mem_end = mmio_start + mmio_len;
1455 /* PCI config space info */
1456 hw->vendor_id = pdev->vendor;
1457 hw->device_id = pdev->device;
1458 hw->revision_id = pdev->revision;
1459 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1460 hw->subsystem_device_id = pdev->subsystem_device;
1462 /* setup the private structure */
1464 /* Copy the default MAC, PHY and NVM function pointers */
1465 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1466 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1467 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1468 /* Initialize skew-specific constants */
1469 err = ei->get_invariants(hw);
1473 #ifdef CONFIG_PCI_IOV
1474 /* since iov functionality isn't critical to base device function we
1475 * can accept failure. If it fails we don't allow iov to be enabled */
1476 if (hw->mac.type == e1000_82576) {
1477 /* 82576 supports a maximum of 7 VFs in addition to the PF */
1478 unsigned int num_vfs = (max_vfs > 7) ? 7 : max_vfs;
1480 unsigned char mac_addr[ETH_ALEN];
1483 adapter->vf_data = kcalloc(num_vfs,
1484 sizeof(struct vf_data_storage),
1486 if (!adapter->vf_data) {
1488 "Could not allocate VF private data - "
1489 "IOV enable failed\n");
1491 err = pci_enable_sriov(pdev, num_vfs);
1493 adapter->vfs_allocated_count = num_vfs;
1494 dev_info(&pdev->dev,
1495 "%d vfs allocated\n",
1498 i < adapter->vfs_allocated_count;
1500 random_ether_addr(mac_addr);
1501 igb_set_vf_mac(adapter, i,
1505 kfree(adapter->vf_data);
1506 adapter->vf_data = NULL;
1513 /* setup the private structure */
1514 err = igb_sw_init(adapter);
1518 igb_get_bus_info_pcie(hw);
1520 hw->phy.autoneg_wait_to_complete = false;
1521 hw->mac.adaptive_ifs = true;
1523 /* Copper options */
1524 if (hw->phy.media_type == e1000_media_type_copper) {
1525 hw->phy.mdix = AUTO_ALL_MODES;
1526 hw->phy.disable_polarity_correction = false;
1527 hw->phy.ms_type = e1000_ms_hw_default;
1530 if (igb_check_reset_block(hw))
1531 dev_info(&pdev->dev,
1532 "PHY reset is blocked due to SOL/IDER session.\n");
1534 netdev->features = NETIF_F_SG |
1536 NETIF_F_HW_VLAN_TX |
1537 NETIF_F_HW_VLAN_RX |
1538 NETIF_F_HW_VLAN_FILTER;
1540 netdev->features |= NETIF_F_IPV6_CSUM;
1541 netdev->features |= NETIF_F_TSO;
1542 netdev->features |= NETIF_F_TSO6;
1544 netdev->features |= NETIF_F_GRO;
1546 netdev->vlan_features |= NETIF_F_TSO;
1547 netdev->vlan_features |= NETIF_F_TSO6;
1548 netdev->vlan_features |= NETIF_F_IP_CSUM;
1549 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1550 netdev->vlan_features |= NETIF_F_SG;
1553 netdev->features |= NETIF_F_HIGHDMA;
1555 if (adapter->hw.mac.type == e1000_82576)
1556 netdev->features |= NETIF_F_SCTP_CSUM;
1558 adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
1560 /* before reading the NVM, reset the controller to put the device in a
1561 * known good starting state */
1562 hw->mac.ops.reset_hw(hw);
1564 /* make sure the NVM is good */
1565 if (igb_validate_nvm_checksum(hw) < 0) {
1566 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1571 /* copy the MAC address out of the NVM */
1572 if (hw->mac.ops.read_mac_addr(hw))
1573 dev_err(&pdev->dev, "NVM Read Error\n");
1575 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1576 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1578 if (!is_valid_ether_addr(netdev->perm_addr)) {
1579 dev_err(&pdev->dev, "Invalid MAC Address\n");
1584 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1585 (unsigned long) adapter);
1586 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1587 (unsigned long) adapter);
1589 INIT_WORK(&adapter->reset_task, igb_reset_task);
1590 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1592 /* Initialize link properties that are user-changeable */
1593 adapter->fc_autoneg = true;
1594 hw->mac.autoneg = true;
1595 hw->phy.autoneg_advertised = 0x2f;
1597 hw->fc.requested_mode = e1000_fc_default;
1598 hw->fc.current_mode = e1000_fc_default;
1600 adapter->itr_setting = IGB_DEFAULT_ITR;
1601 adapter->itr = IGB_START_ITR;
1603 igb_validate_mdi_setting(hw);
1605 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1606 * enable the ACPI Magic Packet filter
1609 if (hw->bus.func == 0)
1610 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1611 else if (hw->bus.func == 1)
1612 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1614 if (eeprom_data & eeprom_apme_mask)
1615 adapter->eeprom_wol |= E1000_WUFC_MAG;
1617 /* now that we have the eeprom settings, apply the special cases where
1618 * the eeprom may be wrong or the board simply won't support wake on
1619 * lan on a particular port */
1620 switch (pdev->device) {
1621 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1622 adapter->eeprom_wol = 0;
1624 case E1000_DEV_ID_82575EB_FIBER_SERDES:
1625 case E1000_DEV_ID_82576_FIBER:
1626 case E1000_DEV_ID_82576_SERDES:
1627 /* Wake events only supported on port A for dual fiber
1628 * regardless of eeprom setting */
1629 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1630 adapter->eeprom_wol = 0;
1632 case E1000_DEV_ID_82576_QUAD_COPPER:
1633 /* if quad port adapter, disable WoL on all but port A */
1634 if (global_quad_port_a != 0)
1635 adapter->eeprom_wol = 0;
1637 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1638 /* Reset for multiple quad port adapters */
1639 if (++global_quad_port_a == 4)
1640 global_quad_port_a = 0;
1644 /* initialize the wol settings based on the eeprom settings */
1645 adapter->wol = adapter->eeprom_wol;
1646 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1648 /* reset the hardware with the new settings */
1651 /* let the f/w know that the h/w is now under the control of the
1653 igb_get_hw_control(adapter);
1655 strcpy(netdev->name, "eth%d");
1656 err = register_netdev(netdev);
1660 /* carrier off reporting is important to ethtool even BEFORE open */
1661 netif_carrier_off(netdev);
1663 #ifdef CONFIG_IGB_DCA
1664 if (dca_add_requester(&pdev->dev) == 0) {
1665 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1666 dev_info(&pdev->dev, "DCA enabled\n");
1667 igb_setup_dca(adapter);
1672 * Initialize hardware timer: we keep it running just in case
1673 * that some program needs it later on.
1675 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1676 adapter->cycles.read = igb_read_clock;
1677 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1678 adapter->cycles.mult = 1;
1679 adapter->cycles.shift = IGB_TSYNC_SHIFT;
1682 IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS * IGB_TSYNC_SCALE);
1685 * Avoid rollover while we initialize by resetting the time counter.
1687 wr32(E1000_SYSTIML, 0x00000000);
1688 wr32(E1000_SYSTIMH, 0x00000000);
1691 * Set registers so that rollover occurs soon to test this.
1693 wr32(E1000_SYSTIML, 0x00000000);
1694 wr32(E1000_SYSTIMH, 0xFF800000);
1697 timecounter_init(&adapter->clock,
1699 ktime_to_ns(ktime_get_real()));
1702 * Synchronize our NIC clock against system wall clock. NIC
1703 * time stamp reading requires ~3us per sample, each sample
1704 * was pretty stable even under load => only require 10
1705 * samples for each offset comparison.
1707 memset(&adapter->compare, 0, sizeof(adapter->compare));
1708 adapter->compare.source = &adapter->clock;
1709 adapter->compare.target = ktime_get_real;
1710 adapter->compare.num_samples = 10;
1711 timecompare_update(&adapter->compare, 0);
1717 "igb: %s: hw %p initialized timer\n",
1718 igb_get_time_str(adapter, buffer),
1723 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1724 /* print bus type/speed/width info */
1725 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1727 ((hw->bus.speed == e1000_bus_speed_2500)
1728 ? "2.5Gb/s" : "unknown"),
1729 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1730 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1731 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1735 igb_read_part_num(hw, &part_num);
1736 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1737 (part_num >> 8), (part_num & 0xff));
1739 dev_info(&pdev->dev,
1740 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1741 adapter->msix_entries ? "MSI-X" :
1742 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
1743 adapter->num_rx_queues, adapter->num_tx_queues);
1748 igb_release_hw_control(adapter);
1750 if (!igb_check_reset_block(hw))
1753 if (hw->flash_address)
1754 iounmap(hw->flash_address);
1756 igb_clear_interrupt_scheme(adapter);
1757 iounmap(hw->hw_addr);
1759 free_netdev(netdev);
1761 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1765 pci_disable_device(pdev);
1770 * igb_remove - Device Removal Routine
1771 * @pdev: PCI device information struct
1773 * igb_remove is called by the PCI subsystem to alert the driver
1774 * that it should release a PCI device. The could be caused by a
1775 * Hot-Plug event, or because the driver is going to be removed from
1778 static void __devexit igb_remove(struct pci_dev *pdev)
1780 struct net_device *netdev = pci_get_drvdata(pdev);
1781 struct igb_adapter *adapter = netdev_priv(netdev);
1782 struct e1000_hw *hw = &adapter->hw;
1784 /* flush_scheduled work may reschedule our watchdog task, so
1785 * explicitly disable watchdog tasks from being rescheduled */
1786 set_bit(__IGB_DOWN, &adapter->state);
1787 del_timer_sync(&adapter->watchdog_timer);
1788 del_timer_sync(&adapter->phy_info_timer);
1790 flush_scheduled_work();
1792 #ifdef CONFIG_IGB_DCA
1793 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
1794 dev_info(&pdev->dev, "DCA disabled\n");
1795 dca_remove_requester(&pdev->dev);
1796 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
1797 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
1801 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1802 * would have already happened in close and is redundant. */
1803 igb_release_hw_control(adapter);
1805 unregister_netdev(netdev);
1807 if (!igb_check_reset_block(&adapter->hw))
1808 igb_reset_phy(&adapter->hw);
1810 igb_clear_interrupt_scheme(adapter);
1812 #ifdef CONFIG_PCI_IOV
1813 /* reclaim resources allocated to VFs */
1814 if (adapter->vf_data) {
1815 /* disable iov and allow time for transactions to clear */
1816 pci_disable_sriov(pdev);
1819 kfree(adapter->vf_data);
1820 adapter->vf_data = NULL;
1821 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1823 dev_info(&pdev->dev, "IOV Disabled\n");
1826 iounmap(hw->hw_addr);
1827 if (hw->flash_address)
1828 iounmap(hw->flash_address);
1829 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1832 free_netdev(netdev);
1834 pci_disable_pcie_error_reporting(pdev);
1836 pci_disable_device(pdev);
1840 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1841 * @adapter: board private structure to initialize
1843 * igb_sw_init initializes the Adapter private data structure.
1844 * Fields are initialized based on PCI device information and
1845 * OS network device settings (MTU size).
1847 static int __devinit igb_sw_init(struct igb_adapter *adapter)
1849 struct e1000_hw *hw = &adapter->hw;
1850 struct net_device *netdev = adapter->netdev;
1851 struct pci_dev *pdev = adapter->pdev;
1853 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1855 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1856 adapter->rx_ring_count = IGB_DEFAULT_RXD;
1857 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1858 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1860 /* This call may decrease the number of queues depending on
1861 * interrupt mode. */
1862 if (igb_init_interrupt_scheme(adapter)) {
1863 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1867 /* Explicitly disable IRQ since the NIC can be in any state. */
1868 igb_irq_disable(adapter);
1870 set_bit(__IGB_DOWN, &adapter->state);
1875 * igb_open - Called when a network interface is made active
1876 * @netdev: network interface device structure
1878 * Returns 0 on success, negative value on failure
1880 * The open entry point is called when a network interface is made
1881 * active by the system (IFF_UP). At this point all resources needed
1882 * for transmit and receive operations are allocated, the interrupt
1883 * handler is registered with the OS, the watchdog timer is started,
1884 * and the stack is notified that the interface is ready.
1886 static int igb_open(struct net_device *netdev)
1888 struct igb_adapter *adapter = netdev_priv(netdev);
1889 struct e1000_hw *hw = &adapter->hw;
1893 /* disallow open during test */
1894 if (test_bit(__IGB_TESTING, &adapter->state))
1897 netif_carrier_off(netdev);
1899 /* allocate transmit descriptors */
1900 err = igb_setup_all_tx_resources(adapter);
1904 /* allocate receive descriptors */
1905 err = igb_setup_all_rx_resources(adapter);
1909 /* e1000_power_up_phy(adapter); */
1911 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1912 if ((adapter->hw.mng_cookie.status &
1913 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
1914 igb_update_mng_vlan(adapter);
1916 /* before we allocate an interrupt, we must be ready to handle it.
1917 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1918 * as soon as we call pci_request_irq, so we have to setup our
1919 * clean_rx handler before we do so. */
1920 igb_configure(adapter);
1922 igb_set_vmolr(hw, adapter->vfs_allocated_count);
1924 err = igb_request_irq(adapter);
1928 /* From here on the code is the same as igb_up() */
1929 clear_bit(__IGB_DOWN, &adapter->state);
1931 for (i = 0; i < adapter->num_q_vectors; i++) {
1932 struct igb_q_vector *q_vector = adapter->q_vector[i];
1933 napi_enable(&q_vector->napi);
1936 /* Clear any pending interrupts. */
1939 igb_irq_enable(adapter);
1941 /* notify VFs that reset has been completed */
1942 if (adapter->vfs_allocated_count) {
1943 u32 reg_data = rd32(E1000_CTRL_EXT);
1944 reg_data |= E1000_CTRL_EXT_PFRSTD;
1945 wr32(E1000_CTRL_EXT, reg_data);
1948 netif_tx_start_all_queues(netdev);
1950 /* Fire a link status change interrupt to start the watchdog. */
1951 wr32(E1000_ICS, E1000_ICS_LSC);
1956 igb_release_hw_control(adapter);
1957 /* e1000_power_down_phy(adapter); */
1958 igb_free_all_rx_resources(adapter);
1960 igb_free_all_tx_resources(adapter);
1968 * igb_close - Disables a network interface
1969 * @netdev: network interface device structure
1971 * Returns 0, this is not allowed to fail
1973 * The close entry point is called when an interface is de-activated
1974 * by the OS. The hardware is still under the driver's control, but
1975 * needs to be disabled. A global MAC reset is issued to stop the
1976 * hardware, and all transmit and receive resources are freed.
1978 static int igb_close(struct net_device *netdev)
1980 struct igb_adapter *adapter = netdev_priv(netdev);
1982 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
1985 igb_free_irq(adapter);
1987 igb_free_all_tx_resources(adapter);
1988 igb_free_all_rx_resources(adapter);
1990 /* kill manageability vlan ID if supported, but not if a vlan with
1991 * the same ID is registered on the host OS (let 8021q kill it) */
1992 if ((adapter->hw.mng_cookie.status &
1993 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
1995 vlan_group_get_device(adapter->vlgrp, adapter->mng_vlan_id)))
1996 igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
2002 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2003 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2005 * Return 0 on success, negative on failure
2007 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2009 struct pci_dev *pdev = tx_ring->pdev;
2012 size = sizeof(struct igb_buffer) * tx_ring->count;
2013 tx_ring->buffer_info = vmalloc(size);
2014 if (!tx_ring->buffer_info)
2016 memset(tx_ring->buffer_info, 0, size);
2018 /* round up to nearest 4K */
2019 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2020 tx_ring->size = ALIGN(tx_ring->size, 4096);
2022 tx_ring->desc = pci_alloc_consistent(pdev, tx_ring->size,
2028 tx_ring->next_to_use = 0;
2029 tx_ring->next_to_clean = 0;
2033 vfree(tx_ring->buffer_info);
2035 "Unable to allocate memory for the transmit descriptor ring\n");
2040 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2041 * (Descriptors) for all queues
2042 * @adapter: board private structure
2044 * Return 0 on success, negative on failure
2046 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2051 for (i = 0; i < adapter->num_tx_queues; i++) {
2052 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
2054 dev_err(&adapter->pdev->dev,
2055 "Allocation for Tx Queue %u failed\n", i);
2056 for (i--; i >= 0; i--)
2057 igb_free_tx_resources(&adapter->tx_ring[i]);
2062 for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
2063 r_idx = i % adapter->num_tx_queues;
2064 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
2070 * igb_setup_tctl - configure the transmit control registers
2071 * @adapter: Board private structure
2073 void igb_setup_tctl(struct igb_adapter *adapter)
2075 struct e1000_hw *hw = &adapter->hw;
2078 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2079 wr32(E1000_TXDCTL(0), 0);
2081 /* Program the Transmit Control Register */
2082 tctl = rd32(E1000_TCTL);
2083 tctl &= ~E1000_TCTL_CT;
2084 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2085 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2087 igb_config_collision_dist(hw);
2089 /* Enable transmits */
2090 tctl |= E1000_TCTL_EN;
2092 wr32(E1000_TCTL, tctl);
2096 * igb_configure_tx_ring - Configure transmit ring after Reset
2097 * @adapter: board private structure
2098 * @ring: tx ring to configure
2100 * Configure a transmit ring after a reset.
2102 void igb_configure_tx_ring(struct igb_adapter *adapter,
2103 struct igb_ring *ring)
2105 struct e1000_hw *hw = &adapter->hw;
2107 u64 tdba = ring->dma;
2108 int reg_idx = ring->reg_idx;
2110 /* disable the queue */
2111 txdctl = rd32(E1000_TXDCTL(reg_idx));
2112 wr32(E1000_TXDCTL(reg_idx),
2113 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2117 wr32(E1000_TDLEN(reg_idx),
2118 ring->count * sizeof(union e1000_adv_tx_desc));
2119 wr32(E1000_TDBAL(reg_idx),
2120 tdba & 0x00000000ffffffffULL);
2121 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2123 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2124 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2125 writel(0, ring->head);
2126 writel(0, ring->tail);
2128 txdctl |= IGB_TX_PTHRESH;
2129 txdctl |= IGB_TX_HTHRESH << 8;
2130 txdctl |= IGB_TX_WTHRESH << 16;
2132 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2133 wr32(E1000_TXDCTL(reg_idx), txdctl);
2137 * igb_configure_tx - Configure transmit Unit after Reset
2138 * @adapter: board private structure
2140 * Configure the Tx unit of the MAC after a reset.
2142 static void igb_configure_tx(struct igb_adapter *adapter)
2146 for (i = 0; i < adapter->num_tx_queues; i++)
2147 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
2151 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2152 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2154 * Returns 0 on success, negative on failure
2156 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2158 struct pci_dev *pdev = rx_ring->pdev;
2161 size = sizeof(struct igb_buffer) * rx_ring->count;
2162 rx_ring->buffer_info = vmalloc(size);
2163 if (!rx_ring->buffer_info)
2165 memset(rx_ring->buffer_info, 0, size);
2167 desc_len = sizeof(union e1000_adv_rx_desc);
2169 /* Round up to nearest 4K */
2170 rx_ring->size = rx_ring->count * desc_len;
2171 rx_ring->size = ALIGN(rx_ring->size, 4096);
2173 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2179 rx_ring->next_to_clean = 0;
2180 rx_ring->next_to_use = 0;
2185 vfree(rx_ring->buffer_info);
2186 dev_err(&pdev->dev, "Unable to allocate memory for "
2187 "the receive descriptor ring\n");
2192 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2193 * (Descriptors) for all queues
2194 * @adapter: board private structure
2196 * Return 0 on success, negative on failure
2198 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2202 for (i = 0; i < adapter->num_rx_queues; i++) {
2203 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
2205 dev_err(&adapter->pdev->dev,
2206 "Allocation for Rx Queue %u failed\n", i);
2207 for (i--; i >= 0; i--)
2208 igb_free_rx_resources(&adapter->rx_ring[i]);
2217 * igb_setup_mrqc - configure the multiple receive queue control registers
2218 * @adapter: Board private structure
2220 static void igb_setup_mrqc(struct igb_adapter *adapter)
2222 struct e1000_hw *hw = &adapter->hw;
2224 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2229 static const u8 rsshash[40] = {
2230 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2231 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2232 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2233 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2235 /* Fill out hash function seeds */
2236 for (j = 0; j < 10; j++) {
2237 u32 rsskey = rsshash[(j * 4)];
2238 rsskey |= rsshash[(j * 4) + 1] << 8;
2239 rsskey |= rsshash[(j * 4) + 2] << 16;
2240 rsskey |= rsshash[(j * 4) + 3] << 24;
2241 array_wr32(E1000_RSSRK(0), j, rsskey);
2244 num_rx_queues = adapter->num_rx_queues;
2246 if (adapter->vfs_allocated_count) {
2247 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2248 switch (hw->mac.type) {
2260 if (hw->mac.type == e1000_82575)
2264 for (j = 0; j < (32 * 4); j++) {
2265 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2267 reta.bytes[j & 3] |= num_rx_queues << shift2;
2269 wr32(E1000_RETA(j >> 2), reta.dword);
2273 * Disable raw packet checksumming so that RSS hash is placed in
2274 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2275 * offloads as they are enabled by default
2277 rxcsum = rd32(E1000_RXCSUM);
2278 rxcsum |= E1000_RXCSUM_PCSD;
2280 if (adapter->hw.mac.type >= e1000_82576)
2281 /* Enable Receive Checksum Offload for SCTP */
2282 rxcsum |= E1000_RXCSUM_CRCOFL;
2284 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2285 wr32(E1000_RXCSUM, rxcsum);
2287 /* If VMDq is enabled then we set the appropriate mode for that, else
2288 * we default to RSS so that an RSS hash is calculated per packet even
2289 * if we are only using one queue */
2290 if (adapter->vfs_allocated_count) {
2291 if (hw->mac.type > e1000_82575) {
2292 /* Set the default pool for the PF's first queue */
2293 u32 vtctl = rd32(E1000_VT_CTL);
2294 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2295 E1000_VT_CTL_DISABLE_DEF_POOL);
2296 vtctl |= adapter->vfs_allocated_count <<
2297 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2298 wr32(E1000_VT_CTL, vtctl);
2300 if (adapter->num_rx_queues > 1)
2301 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2303 mrqc = E1000_MRQC_ENABLE_VMDQ;
2305 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2307 igb_vmm_control(adapter);
2309 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2310 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2311 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2312 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2313 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2314 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2315 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2316 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2318 wr32(E1000_MRQC, mrqc);
2322 * igb_setup_rctl - configure the receive control registers
2323 * @adapter: Board private structure
2325 void igb_setup_rctl(struct igb_adapter *adapter)
2327 struct e1000_hw *hw = &adapter->hw;
2330 rctl = rd32(E1000_RCTL);
2332 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2333 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2335 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2336 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2339 * enable stripping of CRC. It's unlikely this will break BMC
2340 * redirection as it did with e1000. Newer features require
2341 * that the HW strips the CRC.
2343 rctl |= E1000_RCTL_SECRC;
2346 * disable store bad packets and clear size bits.
2348 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2350 /* enable LPE to prevent packets larger than max_frame_size */
2351 rctl |= E1000_RCTL_LPE;
2353 /* disable queue 0 to prevent tail write w/o re-config */
2354 wr32(E1000_RXDCTL(0), 0);
2356 /* Attention!!! For SR-IOV PF driver operations you must enable
2357 * queue drop for all VF and PF queues to prevent head of line blocking
2358 * if an un-trusted VF does not provide descriptors to hardware.
2360 if (adapter->vfs_allocated_count) {
2363 /* set all queue drop enable bits */
2364 wr32(E1000_QDE, ALL_QUEUES);
2366 vmolr = rd32(E1000_VMOLR(adapter->vfs_allocated_count));
2367 if (rctl & E1000_RCTL_LPE)
2368 vmolr |= E1000_VMOLR_LPE;
2369 if (adapter->num_rx_queues > 1)
2370 vmolr |= E1000_VMOLR_RSSE;
2371 wr32(E1000_VMOLR(adapter->vfs_allocated_count), vmolr);
2374 wr32(E1000_RCTL, rctl);
2378 * igb_rlpml_set - set maximum receive packet size
2379 * @adapter: board private structure
2381 * Configure maximum receivable packet size.
2383 static void igb_rlpml_set(struct igb_adapter *adapter)
2385 u32 max_frame_size = adapter->max_frame_size;
2386 struct e1000_hw *hw = &adapter->hw;
2387 u16 pf_id = adapter->vfs_allocated_count;
2390 max_frame_size += VLAN_TAG_SIZE;
2392 /* if vfs are enabled we set RLPML to the largest possible request
2393 * size and set the VMOLR RLPML to the size we need */
2395 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2396 max_frame_size = MAX_STD_JUMBO_FRAME_SIZE + VLAN_TAG_SIZE;
2399 wr32(E1000_RLPML, max_frame_size);
2403 * igb_configure_rx_ring - Configure a receive ring after Reset
2404 * @adapter: board private structure
2405 * @ring: receive ring to be configured
2407 * Configure the Rx unit of the MAC after a reset.
2409 void igb_configure_rx_ring(struct igb_adapter *adapter,
2410 struct igb_ring *ring)
2412 struct e1000_hw *hw = &adapter->hw;
2413 u64 rdba = ring->dma;
2414 int reg_idx = ring->reg_idx;
2417 /* disable the queue */
2418 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2419 wr32(E1000_RXDCTL(reg_idx),
2420 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2422 /* Set DMA base address registers */
2423 wr32(E1000_RDBAL(reg_idx),
2424 rdba & 0x00000000ffffffffULL);
2425 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2426 wr32(E1000_RDLEN(reg_idx),
2427 ring->count * sizeof(union e1000_adv_rx_desc));
2429 /* initialize head and tail */
2430 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2431 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2432 writel(0, ring->head);
2433 writel(0, ring->tail);
2435 /* set descriptor configuration */
2436 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2437 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2438 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2439 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2440 srrctl |= IGB_RXBUFFER_16384 >>
2441 E1000_SRRCTL_BSIZEPKT_SHIFT;
2443 srrctl |= (PAGE_SIZE / 2) >>
2444 E1000_SRRCTL_BSIZEPKT_SHIFT;
2446 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2448 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2449 E1000_SRRCTL_BSIZEPKT_SHIFT;
2450 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2453 wr32(E1000_SRRCTL(reg_idx), srrctl);
2455 /* enable receive descriptor fetching */
2456 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2457 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2458 rxdctl &= 0xFFF00000;
2459 rxdctl |= IGB_RX_PTHRESH;
2460 rxdctl |= IGB_RX_HTHRESH << 8;
2461 rxdctl |= IGB_RX_WTHRESH << 16;
2462 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2466 * igb_configure_rx - Configure receive Unit after Reset
2467 * @adapter: board private structure
2469 * Configure the Rx unit of the MAC after a reset.
2471 static void igb_configure_rx(struct igb_adapter *adapter)
2475 /* set UTA to appropriate mode */
2476 igb_set_uta(adapter);
2478 /* set the correct pool for the PF default MAC address in entry 0 */
2479 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2480 adapter->vfs_allocated_count);
2482 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2483 * the Base and Length of the Rx Descriptor Ring */
2484 for (i = 0; i < adapter->num_rx_queues; i++)
2485 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
2489 * igb_free_tx_resources - Free Tx Resources per Queue
2490 * @tx_ring: Tx descriptor ring for a specific queue
2492 * Free all transmit software resources
2494 void igb_free_tx_resources(struct igb_ring *tx_ring)
2496 igb_clean_tx_ring(tx_ring);
2498 vfree(tx_ring->buffer_info);
2499 tx_ring->buffer_info = NULL;
2501 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2502 tx_ring->desc, tx_ring->dma);
2504 tx_ring->desc = NULL;
2508 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2509 * @adapter: board private structure
2511 * Free all transmit software resources
2513 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2517 for (i = 0; i < adapter->num_tx_queues; i++)
2518 igb_free_tx_resources(&adapter->tx_ring[i]);
2521 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2522 struct igb_buffer *buffer_info)
2524 buffer_info->dma = 0;
2525 if (buffer_info->skb) {
2526 skb_dma_unmap(&tx_ring->pdev->dev,
2529 dev_kfree_skb_any(buffer_info->skb);
2530 buffer_info->skb = NULL;
2532 buffer_info->time_stamp = 0;
2533 /* buffer_info must be completely set up in the transmit path */
2537 * igb_clean_tx_ring - Free Tx Buffers
2538 * @tx_ring: ring to be cleaned
2540 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
2542 struct igb_buffer *buffer_info;
2546 if (!tx_ring->buffer_info)
2548 /* Free all the Tx ring sk_buffs */
2550 for (i = 0; i < tx_ring->count; i++) {
2551 buffer_info = &tx_ring->buffer_info[i];
2552 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
2555 size = sizeof(struct igb_buffer) * tx_ring->count;
2556 memset(tx_ring->buffer_info, 0, size);
2558 /* Zero out the descriptor ring */
2560 memset(tx_ring->desc, 0, tx_ring->size);
2562 tx_ring->next_to_use = 0;
2563 tx_ring->next_to_clean = 0;
2565 writel(0, tx_ring->head);
2566 writel(0, tx_ring->tail);
2570 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2571 * @adapter: board private structure
2573 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2577 for (i = 0; i < adapter->num_tx_queues; i++)
2578 igb_clean_tx_ring(&adapter->tx_ring[i]);
2582 * igb_free_rx_resources - Free Rx Resources
2583 * @rx_ring: ring to clean the resources from
2585 * Free all receive software resources
2587 void igb_free_rx_resources(struct igb_ring *rx_ring)
2589 igb_clean_rx_ring(rx_ring);
2591 vfree(rx_ring->buffer_info);
2592 rx_ring->buffer_info = NULL;
2594 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2595 rx_ring->desc, rx_ring->dma);
2597 rx_ring->desc = NULL;
2601 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2602 * @adapter: board private structure
2604 * Free all receive software resources
2606 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2610 for (i = 0; i < adapter->num_rx_queues; i++)
2611 igb_free_rx_resources(&adapter->rx_ring[i]);
2615 * igb_clean_rx_ring - Free Rx Buffers per Queue
2616 * @rx_ring: ring to free buffers from
2618 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
2620 struct igb_buffer *buffer_info;
2624 if (!rx_ring->buffer_info)
2626 /* Free all the Rx ring sk_buffs */
2627 for (i = 0; i < rx_ring->count; i++) {
2628 buffer_info = &rx_ring->buffer_info[i];
2629 if (buffer_info->dma) {
2630 pci_unmap_single(rx_ring->pdev,
2632 rx_ring->rx_buffer_len,
2633 PCI_DMA_FROMDEVICE);
2634 buffer_info->dma = 0;
2637 if (buffer_info->skb) {
2638 dev_kfree_skb(buffer_info->skb);
2639 buffer_info->skb = NULL;
2641 if (buffer_info->page_dma) {
2642 pci_unmap_page(rx_ring->pdev,
2643 buffer_info->page_dma,
2645 PCI_DMA_FROMDEVICE);
2646 buffer_info->page_dma = 0;
2648 if (buffer_info->page) {
2649 put_page(buffer_info->page);
2650 buffer_info->page = NULL;
2651 buffer_info->page_offset = 0;
2655 size = sizeof(struct igb_buffer) * rx_ring->count;
2656 memset(rx_ring->buffer_info, 0, size);
2658 /* Zero out the descriptor ring */
2659 memset(rx_ring->desc, 0, rx_ring->size);
2661 rx_ring->next_to_clean = 0;
2662 rx_ring->next_to_use = 0;
2664 writel(0, rx_ring->head);
2665 writel(0, rx_ring->tail);
2669 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2670 * @adapter: board private structure
2672 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2676 for (i = 0; i < adapter->num_rx_queues; i++)
2677 igb_clean_rx_ring(&adapter->rx_ring[i]);
2681 * igb_set_mac - Change the Ethernet Address of the NIC
2682 * @netdev: network interface device structure
2683 * @p: pointer to an address structure
2685 * Returns 0 on success, negative on failure
2687 static int igb_set_mac(struct net_device *netdev, void *p)
2689 struct igb_adapter *adapter = netdev_priv(netdev);
2690 struct e1000_hw *hw = &adapter->hw;
2691 struct sockaddr *addr = p;
2693 if (!is_valid_ether_addr(addr->sa_data))
2694 return -EADDRNOTAVAIL;
2696 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2697 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2699 /* set the correct pool for the new PF MAC address in entry 0 */
2700 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2701 adapter->vfs_allocated_count);
2707 * igb_write_mc_addr_list - write multicast addresses to MTA
2708 * @netdev: network interface device structure
2710 * Writes multicast address list to the MTA hash table.
2711 * Returns: -ENOMEM on failure
2712 * 0 on no addresses written
2713 * X on writing X addresses to MTA
2715 static int igb_write_mc_addr_list(struct net_device *netdev)
2717 struct igb_adapter *adapter = netdev_priv(netdev);
2718 struct e1000_hw *hw = &adapter->hw;
2719 struct dev_mc_list *mc_ptr = netdev->mc_list;
2724 if (!netdev->mc_count) {
2725 /* nothing to program, so clear mc list */
2726 igb_update_mc_addr_list(hw, NULL, 0);
2727 igb_restore_vf_multicasts(adapter);
2731 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2735 /* set vmolr receive overflow multicast bit */
2736 vmolr |= E1000_VMOLR_ROMPE;
2738 /* The shared function expects a packed array of only addresses. */
2739 mc_ptr = netdev->mc_list;
2741 for (i = 0; i < netdev->mc_count; i++) {
2744 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2745 mc_ptr = mc_ptr->next;
2747 igb_update_mc_addr_list(hw, mta_list, i);
2750 return netdev->mc_count;
2754 * igb_write_uc_addr_list - write unicast addresses to RAR table
2755 * @netdev: network interface device structure
2757 * Writes unicast address list to the RAR table.
2758 * Returns: -ENOMEM on failure/insufficient address space
2759 * 0 on no addresses written
2760 * X on writing X addresses to the RAR table
2762 static int igb_write_uc_addr_list(struct net_device *netdev)
2764 struct igb_adapter *adapter = netdev_priv(netdev);
2765 struct e1000_hw *hw = &adapter->hw;
2766 unsigned int vfn = adapter->vfs_allocated_count;
2767 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2770 /* return ENOMEM indicating insufficient memory for addresses */
2771 if (netdev->uc.count > rar_entries)
2774 if (netdev->uc.count && rar_entries) {
2775 struct netdev_hw_addr *ha;
2776 list_for_each_entry(ha, &netdev->uc.list, list) {
2779 igb_rar_set_qsel(adapter, ha->addr,
2785 /* write the addresses in reverse order to avoid write combining */
2786 for (; rar_entries > 0 ; rar_entries--) {
2787 wr32(E1000_RAH(rar_entries), 0);
2788 wr32(E1000_RAL(rar_entries), 0);
2796 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2797 * @netdev: network interface device structure
2799 * The set_rx_mode entry point is called whenever the unicast or multicast
2800 * address lists or the network interface flags are updated. This routine is
2801 * responsible for configuring the hardware for proper unicast, multicast,
2802 * promiscuous mode, and all-multi behavior.
2804 static void igb_set_rx_mode(struct net_device *netdev)
2806 struct igb_adapter *adapter = netdev_priv(netdev);
2807 struct e1000_hw *hw = &adapter->hw;
2808 unsigned int vfn = adapter->vfs_allocated_count;
2809 u32 rctl, vmolr = 0;
2812 /* Check for Promiscuous and All Multicast modes */
2813 rctl = rd32(E1000_RCTL);
2815 /* clear the effected bits */
2816 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2818 if (netdev->flags & IFF_PROMISC) {
2819 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2820 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2822 if (netdev->flags & IFF_ALLMULTI) {
2823 rctl |= E1000_RCTL_MPE;
2824 vmolr |= E1000_VMOLR_MPME;
2827 * Write addresses to the MTA, if the attempt fails
2828 * then we should just turn on promiscous mode so
2829 * that we can at least receive multicast traffic
2831 count = igb_write_mc_addr_list(netdev);
2833 rctl |= E1000_RCTL_MPE;
2834 vmolr |= E1000_VMOLR_MPME;
2836 vmolr |= E1000_VMOLR_ROMPE;
2840 * Write addresses to available RAR registers, if there is not
2841 * sufficient space to store all the addresses then enable
2842 * unicast promiscous mode
2844 count = igb_write_uc_addr_list(netdev);
2846 rctl |= E1000_RCTL_UPE;
2847 vmolr |= E1000_VMOLR_ROPE;
2849 rctl |= E1000_RCTL_VFE;
2851 wr32(E1000_RCTL, rctl);
2854 * In order to support SR-IOV and eventually VMDq it is necessary to set
2855 * the VMOLR to enable the appropriate modes. Without this workaround
2856 * we will have issues with VLAN tag stripping not being done for frames
2857 * that are only arriving because we are the default pool
2859 if (hw->mac.type < e1000_82576)
2862 vmolr |= rd32(E1000_VMOLR(vfn)) &
2863 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
2864 wr32(E1000_VMOLR(vfn), vmolr);
2865 igb_restore_vf_multicasts(adapter);
2868 /* Need to wait a few seconds after link up to get diagnostic information from
2870 static void igb_update_phy_info(unsigned long data)
2872 struct igb_adapter *adapter = (struct igb_adapter *) data;
2873 igb_get_phy_info(&adapter->hw);
2877 * igb_has_link - check shared code for link and determine up/down
2878 * @adapter: pointer to driver private info
2880 static bool igb_has_link(struct igb_adapter *adapter)
2882 struct e1000_hw *hw = &adapter->hw;
2883 bool link_active = false;
2886 /* get_link_status is set on LSC (link status) interrupt or
2887 * rx sequence error interrupt. get_link_status will stay
2888 * false until the e1000_check_for_link establishes link
2889 * for copper adapters ONLY
2891 switch (hw->phy.media_type) {
2892 case e1000_media_type_copper:
2893 if (hw->mac.get_link_status) {
2894 ret_val = hw->mac.ops.check_for_link(hw);
2895 link_active = !hw->mac.get_link_status;
2900 case e1000_media_type_internal_serdes:
2901 ret_val = hw->mac.ops.check_for_link(hw);
2902 link_active = hw->mac.serdes_has_link;
2905 case e1000_media_type_unknown:
2913 * igb_watchdog - Timer Call-back
2914 * @data: pointer to adapter cast into an unsigned long
2916 static void igb_watchdog(unsigned long data)
2918 struct igb_adapter *adapter = (struct igb_adapter *)data;
2919 /* Do the rest outside of interrupt context */
2920 schedule_work(&adapter->watchdog_task);
2923 static void igb_watchdog_task(struct work_struct *work)
2925 struct igb_adapter *adapter = container_of(work,
2926 struct igb_adapter, watchdog_task);
2927 struct e1000_hw *hw = &adapter->hw;
2928 struct net_device *netdev = adapter->netdev;
2929 struct igb_ring *tx_ring = adapter->tx_ring;
2933 link = igb_has_link(adapter);
2934 if ((netif_carrier_ok(netdev)) && link)
2938 if (!netif_carrier_ok(netdev)) {
2940 hw->mac.ops.get_speed_and_duplex(&adapter->hw,
2941 &adapter->link_speed,
2942 &adapter->link_duplex);
2944 ctrl = rd32(E1000_CTRL);
2945 /* Links status message must follow this format */
2946 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
2947 "Flow Control: %s\n",
2949 adapter->link_speed,
2950 adapter->link_duplex == FULL_DUPLEX ?
2951 "Full Duplex" : "Half Duplex",
2952 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2953 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2954 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2955 E1000_CTRL_TFCE) ? "TX" : "None")));
2957 /* tweak tx_queue_len according to speed/duplex and
2958 * adjust the timeout factor */
2959 netdev->tx_queue_len = adapter->tx_queue_len;
2960 adapter->tx_timeout_factor = 1;
2961 switch (adapter->link_speed) {
2963 netdev->tx_queue_len = 10;
2964 adapter->tx_timeout_factor = 14;
2967 netdev->tx_queue_len = 100;
2968 /* maybe add some timeout factor ? */
2972 netif_carrier_on(netdev);
2974 igb_ping_all_vfs(adapter);
2976 /* link state has changed, schedule phy info update */
2977 if (!test_bit(__IGB_DOWN, &adapter->state))
2978 mod_timer(&adapter->phy_info_timer,
2979 round_jiffies(jiffies + 2 * HZ));
2982 if (netif_carrier_ok(netdev)) {
2983 adapter->link_speed = 0;
2984 adapter->link_duplex = 0;
2985 /* Links status message must follow this format */
2986 printk(KERN_INFO "igb: %s NIC Link is Down\n",
2988 netif_carrier_off(netdev);
2990 igb_ping_all_vfs(adapter);
2992 /* link state has changed, schedule phy info update */
2993 if (!test_bit(__IGB_DOWN, &adapter->state))
2994 mod_timer(&adapter->phy_info_timer,
2995 round_jiffies(jiffies + 2 * HZ));
3000 igb_update_stats(adapter);
3002 hw->mac.tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
3003 adapter->tpt_old = adapter->stats.tpt;
3004 hw->mac.collision_delta = adapter->stats.colc - adapter->colc_old;
3005 adapter->colc_old = adapter->stats.colc;
3007 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
3008 adapter->gorc_old = adapter->stats.gorc;
3009 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
3010 adapter->gotc_old = adapter->stats.gotc;
3012 igb_update_adaptive(&adapter->hw);
3014 if (!netif_carrier_ok(netdev)) {
3015 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3016 /* We've lost link, so the controller stops DMA,
3017 * but we've got queued Tx work that's never going
3018 * to get done, so reset controller to flush Tx.
3019 * (Do the reset outside of interrupt context). */
3020 adapter->tx_timeout_count++;
3021 schedule_work(&adapter->reset_task);
3022 /* return immediately since reset is imminent */
3027 /* Cause software interrupt to ensure rx ring is cleaned */
3028 if (adapter->msix_entries) {
3030 for (i = 0; i < adapter->num_q_vectors; i++) {
3031 struct igb_q_vector *q_vector = adapter->q_vector[i];
3032 eics |= q_vector->eims_value;
3034 wr32(E1000_EICS, eics);
3036 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3039 /* Force detection of hung controller every watchdog period */
3040 tx_ring->detect_tx_hung = true;
3042 /* Reset the timer */
3043 if (!test_bit(__IGB_DOWN, &adapter->state))
3044 mod_timer(&adapter->watchdog_timer,
3045 round_jiffies(jiffies + 2 * HZ));
3048 enum latency_range {
3052 latency_invalid = 255
3057 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3059 * Stores a new ITR value based on strictly on packet size. This
3060 * algorithm is less sophisticated than that used in igb_update_itr,
3061 * due to the difficulty of synchronizing statistics across multiple
3062 * receive rings. The divisors and thresholds used by this fuction
3063 * were determined based on theoretical maximum wire speed and testing
3064 * data, in order to minimize response time while increasing bulk
3066 * This functionality is controlled by the InterruptThrottleRate module
3067 * parameter (see igb_param.c)
3068 * NOTE: This function is called only when operating in a multiqueue
3069 * receive environment.
3070 * @q_vector: pointer to q_vector
3072 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3074 int new_val = q_vector->itr_val;
3075 int avg_wire_size = 0;
3076 struct igb_adapter *adapter = q_vector->adapter;
3078 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3079 * ints/sec - ITR timer value of 120 ticks.
3081 if (adapter->link_speed != SPEED_1000) {
3086 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3087 struct igb_ring *ring = q_vector->rx_ring;
3088 avg_wire_size = ring->total_bytes / ring->total_packets;
3091 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3092 struct igb_ring *ring = q_vector->tx_ring;
3093 avg_wire_size = max_t(u32, avg_wire_size,
3094 (ring->total_bytes /
3095 ring->total_packets));
3098 /* if avg_wire_size isn't set no work was done */
3102 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3103 avg_wire_size += 24;
3105 /* Don't starve jumbo frames */
3106 avg_wire_size = min(avg_wire_size, 3000);
3108 /* Give a little boost to mid-size frames */
3109 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3110 new_val = avg_wire_size / 3;
3112 new_val = avg_wire_size / 2;
3115 if (new_val != q_vector->itr_val) {
3116 q_vector->itr_val = new_val;
3117 q_vector->set_itr = 1;
3120 if (q_vector->rx_ring) {
3121 q_vector->rx_ring->total_bytes = 0;
3122 q_vector->rx_ring->total_packets = 0;
3124 if (q_vector->tx_ring) {
3125 q_vector->tx_ring->total_bytes = 0;
3126 q_vector->tx_ring->total_packets = 0;
3131 * igb_update_itr - update the dynamic ITR value based on statistics
3132 * Stores a new ITR value based on packets and byte
3133 * counts during the last interrupt. The advantage of per interrupt
3134 * computation is faster updates and more accurate ITR for the current
3135 * traffic pattern. Constants in this function were computed
3136 * based on theoretical maximum wire speed and thresholds were set based
3137 * on testing data as well as attempting to minimize response time
3138 * while increasing bulk throughput.
3139 * this functionality is controlled by the InterruptThrottleRate module
3140 * parameter (see igb_param.c)
3141 * NOTE: These calculations are only valid when operating in a single-
3142 * queue environment.
3143 * @adapter: pointer to adapter
3144 * @itr_setting: current q_vector->itr_val
3145 * @packets: the number of packets during this measurement interval
3146 * @bytes: the number of bytes during this measurement interval
3148 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3149 int packets, int bytes)
3151 unsigned int retval = itr_setting;
3154 goto update_itr_done;
3156 switch (itr_setting) {
3157 case lowest_latency:
3158 /* handle TSO and jumbo frames */
3159 if (bytes/packets > 8000)
3160 retval = bulk_latency;
3161 else if ((packets < 5) && (bytes > 512))
3162 retval = low_latency;
3164 case low_latency: /* 50 usec aka 20000 ints/s */
3165 if (bytes > 10000) {
3166 /* this if handles the TSO accounting */
3167 if (bytes/packets > 8000) {
3168 retval = bulk_latency;
3169 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3170 retval = bulk_latency;
3171 } else if ((packets > 35)) {
3172 retval = lowest_latency;
3174 } else if (bytes/packets > 2000) {
3175 retval = bulk_latency;
3176 } else if (packets <= 2 && bytes < 512) {
3177 retval = lowest_latency;
3180 case bulk_latency: /* 250 usec aka 4000 ints/s */
3181 if (bytes > 25000) {
3183 retval = low_latency;
3184 } else if (bytes < 1500) {
3185 retval = low_latency;
3194 static void igb_set_itr(struct igb_adapter *adapter)
3196 struct igb_q_vector *q_vector = adapter->q_vector[0];
3198 u32 new_itr = q_vector->itr_val;
3200 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3201 if (adapter->link_speed != SPEED_1000) {
3207 adapter->rx_itr = igb_update_itr(adapter,
3209 adapter->rx_ring->total_packets,
3210 adapter->rx_ring->total_bytes);
3212 adapter->tx_itr = igb_update_itr(adapter,
3214 adapter->tx_ring->total_packets,
3215 adapter->tx_ring->total_bytes);
3216 current_itr = max(adapter->rx_itr, adapter->tx_itr);
3218 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3219 if (adapter->itr_setting == 3 && current_itr == lowest_latency)
3220 current_itr = low_latency;
3222 switch (current_itr) {
3223 /* counts and packets in update_itr are dependent on these numbers */
3224 case lowest_latency:
3225 new_itr = 56; /* aka 70,000 ints/sec */
3228 new_itr = 196; /* aka 20,000 ints/sec */
3231 new_itr = 980; /* aka 4,000 ints/sec */
3238 adapter->rx_ring->total_bytes = 0;
3239 adapter->rx_ring->total_packets = 0;
3240 adapter->tx_ring->total_bytes = 0;
3241 adapter->tx_ring->total_packets = 0;
3243 if (new_itr != q_vector->itr_val) {
3244 /* this attempts to bias the interrupt rate towards Bulk
3245 * by adding intermediate steps when interrupt rate is
3247 new_itr = new_itr > q_vector->itr_val ?
3248 max((new_itr * q_vector->itr_val) /
3249 (new_itr + (q_vector->itr_val >> 2)),
3252 /* Don't write the value here; it resets the adapter's
3253 * internal timer, and causes us to delay far longer than
3254 * we should between interrupts. Instead, we write the ITR
3255 * value at the beginning of the next interrupt so the timing
3256 * ends up being correct.
3258 q_vector->itr_val = new_itr;
3259 q_vector->set_itr = 1;
3265 #define IGB_TX_FLAGS_CSUM 0x00000001
3266 #define IGB_TX_FLAGS_VLAN 0x00000002
3267 #define IGB_TX_FLAGS_TSO 0x00000004
3268 #define IGB_TX_FLAGS_IPV4 0x00000008
3269 #define IGB_TX_FLAGS_TSTAMP 0x00000010
3270 #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3271 #define IGB_TX_FLAGS_VLAN_SHIFT 16
3273 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3274 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3276 struct e1000_adv_tx_context_desc *context_desc;
3279 struct igb_buffer *buffer_info;
3280 u32 info = 0, tu_cmd = 0;
3281 u32 mss_l4len_idx, l4len;
3284 if (skb_header_cloned(skb)) {
3285 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3290 l4len = tcp_hdrlen(skb);
3293 if (skb->protocol == htons(ETH_P_IP)) {
3294 struct iphdr *iph = ip_hdr(skb);
3297 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3301 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3302 ipv6_hdr(skb)->payload_len = 0;
3303 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3304 &ipv6_hdr(skb)->daddr,
3308 i = tx_ring->next_to_use;
3310 buffer_info = &tx_ring->buffer_info[i];
3311 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3312 /* VLAN MACLEN IPLEN */
3313 if (tx_flags & IGB_TX_FLAGS_VLAN)
3314 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3315 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3316 *hdr_len += skb_network_offset(skb);
3317 info |= skb_network_header_len(skb);
3318 *hdr_len += skb_network_header_len(skb);
3319 context_desc->vlan_macip_lens = cpu_to_le32(info);
3321 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3322 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3324 if (skb->protocol == htons(ETH_P_IP))
3325 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3326 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3328 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3331 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3332 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3334 /* For 82575, context index must be unique per ring. */
3335 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3336 mss_l4len_idx |= tx_ring->reg_idx << 4;
3338 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3339 context_desc->seqnum_seed = 0;
3341 buffer_info->time_stamp = jiffies;
3342 buffer_info->next_to_watch = i;
3343 buffer_info->dma = 0;
3345 if (i == tx_ring->count)
3348 tx_ring->next_to_use = i;
3353 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3354 struct sk_buff *skb, u32 tx_flags)
3356 struct e1000_adv_tx_context_desc *context_desc;
3357 struct pci_dev *pdev = tx_ring->pdev;
3358 struct igb_buffer *buffer_info;
3359 u32 info = 0, tu_cmd = 0;
3362 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3363 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3364 i = tx_ring->next_to_use;
3365 buffer_info = &tx_ring->buffer_info[i];
3366 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3368 if (tx_flags & IGB_TX_FLAGS_VLAN)
3369 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3370 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3371 if (skb->ip_summed == CHECKSUM_PARTIAL)
3372 info |= skb_network_header_len(skb);
3374 context_desc->vlan_macip_lens = cpu_to_le32(info);
3376 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3378 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3381 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3382 const struct vlan_ethhdr *vhdr =
3383 (const struct vlan_ethhdr*)skb->data;
3385 protocol = vhdr->h_vlan_encapsulated_proto;
3387 protocol = skb->protocol;
3391 case cpu_to_be16(ETH_P_IP):
3392 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3393 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3394 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3395 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3396 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3398 case cpu_to_be16(ETH_P_IPV6):
3399 /* XXX what about other V6 headers?? */
3400 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3401 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3402 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3403 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3406 if (unlikely(net_ratelimit()))
3407 dev_warn(&pdev->dev,
3408 "partial checksum but proto=%x!\n",
3414 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3415 context_desc->seqnum_seed = 0;
3416 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3417 context_desc->mss_l4len_idx =
3418 cpu_to_le32(tx_ring->reg_idx << 4);
3420 buffer_info->time_stamp = jiffies;
3421 buffer_info->next_to_watch = i;
3422 buffer_info->dma = 0;
3425 if (i == tx_ring->count)
3427 tx_ring->next_to_use = i;
3434 #define IGB_MAX_TXD_PWR 16
3435 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3437 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3440 struct igb_buffer *buffer_info;
3441 struct pci_dev *pdev = tx_ring->pdev;
3442 unsigned int len = skb_headlen(skb);
3443 unsigned int count = 0, i;
3447 i = tx_ring->next_to_use;
3449 if (skb_dma_map(&pdev->dev, skb, DMA_TO_DEVICE)) {
3450 dev_err(&pdev->dev, "TX DMA map failed\n");
3454 map = skb_shinfo(skb)->dma_maps;
3456 buffer_info = &tx_ring->buffer_info[i];
3457 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3458 buffer_info->length = len;
3459 /* set time_stamp *before* dma to help avoid a possible race */
3460 buffer_info->time_stamp = jiffies;
3461 buffer_info->next_to_watch = i;
3462 buffer_info->dma = skb_shinfo(skb)->dma_head;
3464 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3465 struct skb_frag_struct *frag;
3468 if (i == tx_ring->count)
3471 frag = &skb_shinfo(skb)->frags[f];
3474 buffer_info = &tx_ring->buffer_info[i];
3475 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3476 buffer_info->length = len;
3477 buffer_info->time_stamp = jiffies;
3478 buffer_info->next_to_watch = i;
3479 buffer_info->dma = map[count];
3483 tx_ring->buffer_info[i].skb = skb;
3484 tx_ring->buffer_info[first].next_to_watch = i;
3489 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3490 int tx_flags, int count, u32 paylen,
3493 union e1000_adv_tx_desc *tx_desc = NULL;
3494 struct igb_buffer *buffer_info;
3495 u32 olinfo_status = 0, cmd_type_len;
3498 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3499 E1000_ADVTXD_DCMD_DEXT);
3501 if (tx_flags & IGB_TX_FLAGS_VLAN)
3502 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3504 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3505 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3507 if (tx_flags & IGB_TX_FLAGS_TSO) {
3508 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3510 /* insert tcp checksum */
3511 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3513 /* insert ip checksum */
3514 if (tx_flags & IGB_TX_FLAGS_IPV4)
3515 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3517 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3518 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3521 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3522 (tx_flags & (IGB_TX_FLAGS_CSUM |
3524 IGB_TX_FLAGS_VLAN)))
3525 olinfo_status |= tx_ring->reg_idx << 4;
3527 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3529 i = tx_ring->next_to_use;
3531 buffer_info = &tx_ring->buffer_info[i];
3532 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3533 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3534 tx_desc->read.cmd_type_len =
3535 cpu_to_le32(cmd_type_len | buffer_info->length);
3536 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3538 if (i == tx_ring->count)
3542 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
3543 /* Force memory writes to complete before letting h/w
3544 * know there are new descriptors to fetch. (Only
3545 * applicable for weak-ordered memory model archs,
3546 * such as IA-64). */
3549 tx_ring->next_to_use = i;
3550 writel(i, tx_ring->tail);
3551 /* we need this if more than one processor can write to our tail
3552 * at a time, it syncronizes IO on IA64/Altix systems */
3556 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3558 struct net_device *netdev = tx_ring->netdev;
3560 netif_stop_subqueue(netdev, tx_ring->queue_index);
3562 /* Herbert's original patch had:
3563 * smp_mb__after_netif_stop_queue();
3564 * but since that doesn't exist yet, just open code it. */
3567 /* We need to check again in a case another CPU has just
3568 * made room available. */
3569 if (igb_desc_unused(tx_ring) < size)
3573 netif_wake_subqueue(netdev, tx_ring->queue_index);
3574 tx_ring->tx_stats.restart_queue++;
3578 static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3580 if (igb_desc_unused(tx_ring) >= size)
3582 return __igb_maybe_stop_tx(tx_ring, size);
3585 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3586 struct igb_ring *tx_ring)
3588 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
3590 unsigned int tx_flags = 0;
3594 union skb_shared_tx *shtx;
3596 /* need: 1 descriptor per page,
3597 * + 2 desc gap to keep tail from touching head,
3598 * + 1 desc for skb->data,
3599 * + 1 desc for context descriptor,
3600 * otherwise try next time */
3601 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
3602 /* this is a hard error */
3603 return NETDEV_TX_BUSY;
3607 * TODO: check that there currently is no other packet with
3608 * time stamping in the queue
3610 * When doing time stamping, keep the connection to the socket
3611 * a while longer: it is still needed by skb_hwtstamp_tx(),
3612 * called either in igb_tx_hwtstamp() or by our caller when
3613 * doing software time stamping.
3616 if (unlikely(shtx->hardware)) {
3617 shtx->in_progress = 1;
3618 tx_flags |= IGB_TX_FLAGS_TSTAMP;
3621 if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
3622 tx_flags |= IGB_TX_FLAGS_VLAN;
3623 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3626 if (skb->protocol == htons(ETH_P_IP))
3627 tx_flags |= IGB_TX_FLAGS_IPV4;
3629 first = tx_ring->next_to_use;
3630 if (skb_is_gso(skb)) {
3631 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
3633 dev_kfree_skb_any(skb);
3634 return NETDEV_TX_OK;
3639 tx_flags |= IGB_TX_FLAGS_TSO;
3640 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
3641 (skb->ip_summed == CHECKSUM_PARTIAL))
3642 tx_flags |= IGB_TX_FLAGS_CSUM;
3645 * count reflects descriptors mapped, if 0 then mapping error
3646 * has occured and we need to rewind the descriptor queue
3648 count = igb_tx_map_adv(tx_ring, skb, first);
3651 dev_kfree_skb_any(skb);
3652 tx_ring->buffer_info[first].time_stamp = 0;
3653 tx_ring->next_to_use = first;
3654 return NETDEV_TX_OK;
3657 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3659 /* Make sure there is space in the ring for the next send. */
3660 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
3662 return NETDEV_TX_OK;
3665 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3666 struct net_device *netdev)
3668 struct igb_adapter *adapter = netdev_priv(netdev);
3669 struct igb_ring *tx_ring;
3672 if (test_bit(__IGB_DOWN, &adapter->state)) {
3673 dev_kfree_skb_any(skb);
3674 return NETDEV_TX_OK;
3677 if (skb->len <= 0) {
3678 dev_kfree_skb_any(skb);
3679 return NETDEV_TX_OK;
3682 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
3683 tx_ring = adapter->multi_tx_table[r_idx];
3685 /* This goes back to the question of how to logically map a tx queue
3686 * to a flow. Right now, performance is impacted slightly negatively
3687 * if using multiple tx queues. If the stack breaks away from a
3688 * single qdisc implementation, we can look at this again. */
3689 return igb_xmit_frame_ring_adv(skb, tx_ring);
3693 * igb_tx_timeout - Respond to a Tx Hang
3694 * @netdev: network interface device structure
3696 static void igb_tx_timeout(struct net_device *netdev)
3698 struct igb_adapter *adapter = netdev_priv(netdev);
3699 struct e1000_hw *hw = &adapter->hw;
3701 /* Do the reset outside of interrupt context */
3702 adapter->tx_timeout_count++;
3703 schedule_work(&adapter->reset_task);
3705 (adapter->eims_enable_mask & ~adapter->eims_other));
3708 static void igb_reset_task(struct work_struct *work)
3710 struct igb_adapter *adapter;
3711 adapter = container_of(work, struct igb_adapter, reset_task);
3713 igb_reinit_locked(adapter);
3717 * igb_get_stats - Get System Network Statistics
3718 * @netdev: network interface device structure
3720 * Returns the address of the device statistics structure.
3721 * The statistics are actually updated from the timer callback.
3723 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
3725 /* only return the current stats */
3726 return &netdev->stats;
3730 * igb_change_mtu - Change the Maximum Transfer Unit
3731 * @netdev: network interface device structure
3732 * @new_mtu: new value for maximum frame size
3734 * Returns 0 on success, negative on failure
3736 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3738 struct igb_adapter *adapter = netdev_priv(netdev);
3739 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3740 u32 rx_buffer_len, i;
3742 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
3743 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3744 dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
3748 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3749 dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
3753 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3756 /* igb_down has a dependency on max_frame_size */
3757 adapter->max_frame_size = max_frame;
3758 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3759 * means we reserve 2 more, this pushes us to allocate from the next
3761 * i.e. RXBUFFER_2048 --> size-4096 slab
3764 if (max_frame <= IGB_RXBUFFER_1024)
3765 rx_buffer_len = IGB_RXBUFFER_1024;
3766 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
3767 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3769 rx_buffer_len = IGB_RXBUFFER_128;
3771 if (netif_running(netdev))
3774 dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
3775 netdev->mtu, new_mtu);
3776 netdev->mtu = new_mtu;
3778 for (i = 0; i < adapter->num_rx_queues; i++)
3779 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3781 if (netif_running(netdev))
3786 clear_bit(__IGB_RESETTING, &adapter->state);
3792 * igb_update_stats - Update the board statistics counters
3793 * @adapter: board private structure
3796 void igb_update_stats(struct igb_adapter *adapter)
3798 struct net_device *netdev = adapter->netdev;
3799 struct e1000_hw *hw = &adapter->hw;
3800 struct pci_dev *pdev = adapter->pdev;
3803 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3806 * Prevent stats update while adapter is being reset, or if the pci
3807 * connection is down.
3809 if (adapter->link_speed == 0)
3811 if (pci_channel_offline(pdev))
3814 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3815 adapter->stats.gprc += rd32(E1000_GPRC);
3816 adapter->stats.gorc += rd32(E1000_GORCL);
3817 rd32(E1000_GORCH); /* clear GORCL */
3818 adapter->stats.bprc += rd32(E1000_BPRC);
3819 adapter->stats.mprc += rd32(E1000_MPRC);
3820 adapter->stats.roc += rd32(E1000_ROC);
3822 adapter->stats.prc64 += rd32(E1000_PRC64);
3823 adapter->stats.prc127 += rd32(E1000_PRC127);
3824 adapter->stats.prc255 += rd32(E1000_PRC255);
3825 adapter->stats.prc511 += rd32(E1000_PRC511);
3826 adapter->stats.prc1023 += rd32(E1000_PRC1023);
3827 adapter->stats.prc1522 += rd32(E1000_PRC1522);
3828 adapter->stats.symerrs += rd32(E1000_SYMERRS);
3829 adapter->stats.sec += rd32(E1000_SEC);
3831 adapter->stats.mpc += rd32(E1000_MPC);
3832 adapter->stats.scc += rd32(E1000_SCC);
3833 adapter->stats.ecol += rd32(E1000_ECOL);
3834 adapter->stats.mcc += rd32(E1000_MCC);
3835 adapter->stats.latecol += rd32(E1000_LATECOL);
3836 adapter->stats.dc += rd32(E1000_DC);
3837 adapter->stats.rlec += rd32(E1000_RLEC);
3838 adapter->stats.xonrxc += rd32(E1000_XONRXC);
3839 adapter->stats.xontxc += rd32(E1000_XONTXC);
3840 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
3841 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
3842 adapter->stats.fcruc += rd32(E1000_FCRUC);
3843 adapter->stats.gptc += rd32(E1000_GPTC);
3844 adapter->stats.gotc += rd32(E1000_GOTCL);
3845 rd32(E1000_GOTCH); /* clear GOTCL */
3846 adapter->stats.rnbc += rd32(E1000_RNBC);
3847 adapter->stats.ruc += rd32(E1000_RUC);
3848 adapter->stats.rfc += rd32(E1000_RFC);
3849 adapter->stats.rjc += rd32(E1000_RJC);
3850 adapter->stats.tor += rd32(E1000_TORH);
3851 adapter->stats.tot += rd32(E1000_TOTH);
3852 adapter->stats.tpr += rd32(E1000_TPR);
3854 adapter->stats.ptc64 += rd32(E1000_PTC64);
3855 adapter->stats.ptc127 += rd32(E1000_PTC127);
3856 adapter->stats.ptc255 += rd32(E1000_PTC255);
3857 adapter->stats.ptc511 += rd32(E1000_PTC511);
3858 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
3859 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
3861 adapter->stats.mptc += rd32(E1000_MPTC);
3862 adapter->stats.bptc += rd32(E1000_BPTC);
3864 /* used for adaptive IFS */
3866 hw->mac.tx_packet_delta = rd32(E1000_TPT);
3867 adapter->stats.tpt += hw->mac.tx_packet_delta;
3868 hw->mac.collision_delta = rd32(E1000_COLC);
3869 adapter->stats.colc += hw->mac.collision_delta;
3871 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
3872 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3873 adapter->stats.tncrs += rd32(E1000_TNCRS);
3874 adapter->stats.tsctc += rd32(E1000_TSCTC);
3875 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
3877 adapter->stats.iac += rd32(E1000_IAC);
3878 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
3879 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
3880 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
3881 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
3882 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
3883 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
3884 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
3885 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
3887 /* Fill out the OS statistics structure */
3888 netdev->stats.multicast = adapter->stats.mprc;
3889 netdev->stats.collisions = adapter->stats.colc;
3893 if (hw->mac.type != e1000_82575) {
3895 u64 rqdpc_total = 0;
3897 /* Read out drops stats per RX queue. Notice RQDPC (Receive
3898 * Queue Drop Packet Count) stats only gets incremented, if
3899 * the DROP_EN but it set (in the SRRCTL register for that
3900 * queue). If DROP_EN bit is NOT set, then the some what
3901 * equivalent count is stored in RNBC (not per queue basis).
3902 * Also note the drop count is due to lack of available
3905 for (i = 0; i < adapter->num_rx_queues; i++) {
3906 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0xFFF;
3907 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3908 rqdpc_total += adapter->rx_ring[i].rx_stats.drops;
3910 netdev->stats.rx_fifo_errors = rqdpc_total;
3913 /* Note RNBC (Receive No Buffers Count) is an not an exact
3914 * drop count as the hardware FIFO might save the day. Thats
3915 * one of the reason for saving it in rx_fifo_errors, as its
3916 * potentially not a true drop.
3918 netdev->stats.rx_fifo_errors += adapter->stats.rnbc;
3920 /* RLEC on some newer hardware can be incorrect so build
3921 * our own version based on RUC and ROC */
3922 netdev->stats.rx_errors = adapter->stats.rxerrc +
3923 adapter->stats.crcerrs + adapter->stats.algnerrc +
3924 adapter->stats.ruc + adapter->stats.roc +
3925 adapter->stats.cexterr;
3926 netdev->stats.rx_length_errors = adapter->stats.ruc +
3928 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3929 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3930 netdev->stats.rx_missed_errors = adapter->stats.mpc;
3933 netdev->stats.tx_errors = adapter->stats.ecol +
3934 adapter->stats.latecol;
3935 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3936 netdev->stats.tx_window_errors = adapter->stats.latecol;
3937 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3939 /* Tx Dropped needs to be maintained elsewhere */
3942 if (hw->phy.media_type == e1000_media_type_copper) {
3943 if ((adapter->link_speed == SPEED_1000) &&
3944 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3945 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3946 adapter->phy_stats.idle_errors += phy_tmp;
3950 /* Management Stats */
3951 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3952 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3953 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3956 static irqreturn_t igb_msix_other(int irq, void *data)
3958 struct igb_adapter *adapter = data;
3959 struct e1000_hw *hw = &adapter->hw;
3960 u32 icr = rd32(E1000_ICR);
3961 /* reading ICR causes bit 31 of EICR to be cleared */
3963 if (icr & E1000_ICR_DOUTSYNC) {
3964 /* HW is reporting DMA is out of sync */
3965 adapter->stats.doosync++;
3968 /* Check for a mailbox event */
3969 if (icr & E1000_ICR_VMMB)
3970 igb_msg_task(adapter);
3972 if (icr & E1000_ICR_LSC) {
3973 hw->mac.get_link_status = 1;
3974 /* guard against interrupt when we're going down */
3975 if (!test_bit(__IGB_DOWN, &adapter->state))
3976 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3979 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_VMMB);
3980 wr32(E1000_EIMS, adapter->eims_other);
3985 static void igb_write_itr(struct igb_q_vector *q_vector)
3987 u32 itr_val = q_vector->itr_val & 0x7FFC;
3989 if (!q_vector->set_itr)
3995 if (q_vector->itr_shift)
3996 itr_val |= itr_val << q_vector->itr_shift;
3998 itr_val |= 0x8000000;
4000 writel(itr_val, q_vector->itr_register);
4001 q_vector->set_itr = 0;
4004 static irqreturn_t igb_msix_ring(int irq, void *data)
4006 struct igb_q_vector *q_vector = data;
4008 /* Write the ITR value calculated from the previous interrupt. */
4009 igb_write_itr(q_vector);
4011 napi_schedule(&q_vector->napi);
4016 #ifdef CONFIG_IGB_DCA
4017 static void igb_update_dca(struct igb_q_vector *q_vector)
4019 struct igb_adapter *adapter = q_vector->adapter;
4020 struct e1000_hw *hw = &adapter->hw;
4021 int cpu = get_cpu();
4023 if (q_vector->cpu == cpu)
4026 if (q_vector->tx_ring) {
4027 int q = q_vector->tx_ring->reg_idx;
4028 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4029 if (hw->mac.type == e1000_82575) {
4030 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4031 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4033 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4034 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4035 E1000_DCA_TXCTRL_CPUID_SHIFT;
4037 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4038 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4040 if (q_vector->rx_ring) {
4041 int q = q_vector->rx_ring->reg_idx;
4042 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4043 if (hw->mac.type == e1000_82575) {
4044 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4045 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4047 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4048 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4049 E1000_DCA_RXCTRL_CPUID_SHIFT;
4051 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4052 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4053 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4054 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4056 q_vector->cpu = cpu;
4061 static void igb_setup_dca(struct igb_adapter *adapter)
4063 struct e1000_hw *hw = &adapter->hw;
4066 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4069 /* Always use CB2 mode, difference is masked in the CB driver. */
4070 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4072 for (i = 0; i < adapter->num_q_vectors; i++) {
4073 struct igb_q_vector *q_vector = adapter->q_vector[i];
4075 igb_update_dca(q_vector);
4079 static int __igb_notify_dca(struct device *dev, void *data)
4081 struct net_device *netdev = dev_get_drvdata(dev);
4082 struct igb_adapter *adapter = netdev_priv(netdev);
4083 struct e1000_hw *hw = &adapter->hw;
4084 unsigned long event = *(unsigned long *)data;
4087 case DCA_PROVIDER_ADD:
4088 /* if already enabled, don't do it again */
4089 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4091 /* Always use CB2 mode, difference is masked
4092 * in the CB driver. */
4093 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4094 if (dca_add_requester(dev) == 0) {
4095 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4096 dev_info(&adapter->pdev->dev, "DCA enabled\n");
4097 igb_setup_dca(adapter);
4100 /* Fall Through since DCA is disabled. */
4101 case DCA_PROVIDER_REMOVE:
4102 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4103 /* without this a class_device is left
4104 * hanging around in the sysfs model */
4105 dca_remove_requester(dev);
4106 dev_info(&adapter->pdev->dev, "DCA disabled\n");
4107 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4108 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4116 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4121 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4124 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4126 #endif /* CONFIG_IGB_DCA */
4128 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4130 struct e1000_hw *hw = &adapter->hw;
4134 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4135 ping = E1000_PF_CONTROL_MSG;
4136 if (adapter->vf_data[i].clear_to_send)
4137 ping |= E1000_VT_MSGTYPE_CTS;
4138 igb_write_mbx(hw, &ping, 1, i);
4142 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4143 u32 *msgbuf, u32 vf)
4145 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4146 u16 *hash_list = (u16 *)&msgbuf[1];
4147 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4150 /* only up to 30 hash values supported */
4154 /* salt away the number of multi cast addresses assigned
4155 * to this VF for later use to restore when the PF multi cast
4158 vf_data->num_vf_mc_hashes = n;
4160 /* VFs are limited to using the MTA hash table for their multicast
4162 for (i = 0; i < n; i++)
4163 vf_data->vf_mc_hashes[i] = hash_list[i];
4165 /* Flush and reset the mta with the new values */
4166 igb_set_rx_mode(adapter->netdev);
4171 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4173 struct e1000_hw *hw = &adapter->hw;
4174 struct vf_data_storage *vf_data;
4177 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4178 vf_data = &adapter->vf_data[i];
4179 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4180 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4184 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4186 struct e1000_hw *hw = &adapter->hw;
4187 u32 pool_mask, reg, vid;
4190 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4192 /* Find the vlan filter for this id */
4193 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4194 reg = rd32(E1000_VLVF(i));
4196 /* remove the vf from the pool */
4199 /* if pool is empty then remove entry from vfta */
4200 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4201 (reg & E1000_VLVF_VLANID_ENABLE)) {
4203 vid = reg & E1000_VLVF_VLANID_MASK;
4204 igb_vfta_set(hw, vid, false);
4207 wr32(E1000_VLVF(i), reg);
4210 adapter->vf_data[vf].vlans_enabled = 0;
4213 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4215 struct e1000_hw *hw = &adapter->hw;
4218 /* It is an error to call this function when VFs are not enabled */
4219 if (!adapter->vfs_allocated_count)
4222 /* Find the vlan filter for this id */
4223 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4224 reg = rd32(E1000_VLVF(i));
4225 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4226 vid == (reg & E1000_VLVF_VLANID_MASK))
4231 if (i == E1000_VLVF_ARRAY_SIZE) {
4232 /* Did not find a matching VLAN ID entry that was
4233 * enabled. Search for a free filter entry, i.e.
4234 * one without the enable bit set
4236 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4237 reg = rd32(E1000_VLVF(i));
4238 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4242 if (i < E1000_VLVF_ARRAY_SIZE) {
4243 /* Found an enabled/available entry */
4244 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4246 /* if !enabled we need to set this up in vfta */
4247 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4248 /* add VID to filter table, if bit already set
4249 * PF must have added it outside of table */
4250 if (igb_vfta_set(hw, vid, true))
4251 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT +
4252 adapter->vfs_allocated_count);
4253 reg |= E1000_VLVF_VLANID_ENABLE;
4255 reg &= ~E1000_VLVF_VLANID_MASK;
4258 wr32(E1000_VLVF(i), reg);
4260 /* do not modify RLPML for PF devices */
4261 if (vf >= adapter->vfs_allocated_count)
4264 if (!adapter->vf_data[vf].vlans_enabled) {
4266 reg = rd32(E1000_VMOLR(vf));
4267 size = reg & E1000_VMOLR_RLPML_MASK;
4269 reg &= ~E1000_VMOLR_RLPML_MASK;
4271 wr32(E1000_VMOLR(vf), reg);
4273 adapter->vf_data[vf].vlans_enabled++;
4278 if (i < E1000_VLVF_ARRAY_SIZE) {
4279 /* remove vf from the pool */
4280 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4281 /* if pool is empty then remove entry from vfta */
4282 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4284 igb_vfta_set(hw, vid, false);
4286 wr32(E1000_VLVF(i), reg);
4288 /* do not modify RLPML for PF devices */
4289 if (vf >= adapter->vfs_allocated_count)
4292 adapter->vf_data[vf].vlans_enabled--;
4293 if (!adapter->vf_data[vf].vlans_enabled) {
4295 reg = rd32(E1000_VMOLR(vf));
4296 size = reg & E1000_VMOLR_RLPML_MASK;
4298 reg &= ~E1000_VMOLR_RLPML_MASK;
4300 wr32(E1000_VMOLR(vf), reg);
4308 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4310 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4311 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4313 return igb_vlvf_set(adapter, vid, add, vf);
4316 static inline void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4318 struct e1000_hw *hw = &adapter->hw;
4320 /* disable mailbox functionality for vf */
4321 adapter->vf_data[vf].clear_to_send = false;
4323 /* reset offloads to defaults */
4324 igb_set_vmolr(hw, vf);
4326 /* reset vlans for device */
4327 igb_clear_vf_vfta(adapter, vf);
4329 /* reset multicast table array for vf */
4330 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4332 /* Flush and reset the mta with the new values */
4333 igb_set_rx_mode(adapter->netdev);
4336 static inline void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4338 struct e1000_hw *hw = &adapter->hw;
4339 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4340 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4342 u8 *addr = (u8 *)(&msgbuf[1]);
4344 /* process all the same items cleared in a function level reset */
4345 igb_vf_reset_event(adapter, vf);
4347 /* set vf mac address */
4348 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4350 /* enable transmit and receive for vf */
4351 reg = rd32(E1000_VFTE);
4352 wr32(E1000_VFTE, reg | (1 << vf));
4353 reg = rd32(E1000_VFRE);
4354 wr32(E1000_VFRE, reg | (1 << vf));
4356 /* enable mailbox functionality for vf */
4357 adapter->vf_data[vf].clear_to_send = true;
4359 /* reply to reset with ack and vf mac address */
4360 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4361 memcpy(addr, vf_mac, 6);
4362 igb_write_mbx(hw, msgbuf, 3, vf);
4365 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4367 unsigned char *addr = (char *)&msg[1];
4370 if (is_valid_ether_addr(addr))
4371 err = igb_set_vf_mac(adapter, vf, addr);
4377 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4379 struct e1000_hw *hw = &adapter->hw;
4380 u32 msg = E1000_VT_MSGTYPE_NACK;
4382 /* if device isn't clear to send it shouldn't be reading either */
4383 if (!adapter->vf_data[vf].clear_to_send)
4384 igb_write_mbx(hw, &msg, 1, vf);
4388 static void igb_msg_task(struct igb_adapter *adapter)
4390 struct e1000_hw *hw = &adapter->hw;
4393 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4394 /* process any reset requests */
4395 if (!igb_check_for_rst(hw, vf)) {
4396 adapter->vf_data[vf].clear_to_send = false;
4397 igb_vf_reset_event(adapter, vf);
4400 /* process any messages pending */
4401 if (!igb_check_for_msg(hw, vf))
4402 igb_rcv_msg_from_vf(adapter, vf);
4404 /* process any acks */
4405 if (!igb_check_for_ack(hw, vf))
4406 igb_rcv_ack_from_vf(adapter, vf);
4411 static int igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4413 u32 mbx_size = E1000_VFMAILBOX_SIZE;
4414 u32 msgbuf[mbx_size];
4415 struct e1000_hw *hw = &adapter->hw;
4418 retval = igb_read_mbx(hw, msgbuf, mbx_size, vf);
4421 dev_err(&adapter->pdev->dev,
4422 "Error receiving message from VF\n");
4424 /* this is a message we already processed, do nothing */
4425 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
4429 * until the vf completes a reset it should not be
4430 * allowed to start any configuration.
4433 if (msgbuf[0] == E1000_VF_RESET) {
4434 igb_vf_reset_msg(adapter, vf);
4439 if (!adapter->vf_data[vf].clear_to_send) {
4440 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4441 igb_write_mbx(hw, msgbuf, 1, vf);
4445 switch ((msgbuf[0] & 0xFFFF)) {
4446 case E1000_VF_SET_MAC_ADDR:
4447 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4449 case E1000_VF_SET_MULTICAST:
4450 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4452 case E1000_VF_SET_LPE:
4453 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4455 case E1000_VF_SET_VLAN:
4456 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4459 dev_err(&adapter->pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4464 /* notify the VF of the results of what it sent us */
4466 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4468 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4470 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4472 igb_write_mbx(hw, msgbuf, 1, vf);
4478 * igb_set_uta - Set unicast filter table address
4479 * @adapter: board private structure
4481 * The unicast table address is a register array of 32-bit registers.
4482 * The table is meant to be used in a way similar to how the MTA is used
4483 * however due to certain limitations in the hardware it is necessary to
4484 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4485 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4487 static void igb_set_uta(struct igb_adapter *adapter)
4489 struct e1000_hw *hw = &adapter->hw;
4492 /* The UTA table only exists on 82576 hardware and newer */
4493 if (hw->mac.type < e1000_82576)
4496 /* we only need to do this if VMDq is enabled */
4497 if (!adapter->vfs_allocated_count)
4500 for (i = 0; i < hw->mac.uta_reg_count; i++)
4501 array_wr32(E1000_UTA, i, ~0);
4505 * igb_intr_msi - Interrupt Handler
4506 * @irq: interrupt number
4507 * @data: pointer to a network interface device structure
4509 static irqreturn_t igb_intr_msi(int irq, void *data)
4511 struct igb_adapter *adapter = data;
4512 struct igb_q_vector *q_vector = adapter->q_vector[0];
4513 struct e1000_hw *hw = &adapter->hw;
4514 /* read ICR disables interrupts using IAM */
4515 u32 icr = rd32(E1000_ICR);
4517 igb_write_itr(q_vector);
4519 if (icr & E1000_ICR_DOUTSYNC) {
4520 /* HW is reporting DMA is out of sync */
4521 adapter->stats.doosync++;
4524 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4525 hw->mac.get_link_status = 1;
4526 if (!test_bit(__IGB_DOWN, &adapter->state))
4527 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4530 napi_schedule(&q_vector->napi);
4536 * igb_intr - Legacy Interrupt Handler
4537 * @irq: interrupt number
4538 * @data: pointer to a network interface device structure
4540 static irqreturn_t igb_intr(int irq, void *data)
4542 struct igb_adapter *adapter = data;
4543 struct igb_q_vector *q_vector = adapter->q_vector[0];
4544 struct e1000_hw *hw = &adapter->hw;
4545 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4546 * need for the IMC write */
4547 u32 icr = rd32(E1000_ICR);
4549 return IRQ_NONE; /* Not our interrupt */
4551 igb_write_itr(q_vector);
4553 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4554 * not set, then the adapter didn't send an interrupt */
4555 if (!(icr & E1000_ICR_INT_ASSERTED))
4558 if (icr & E1000_ICR_DOUTSYNC) {
4559 /* HW is reporting DMA is out of sync */
4560 adapter->stats.doosync++;
4563 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4564 hw->mac.get_link_status = 1;
4565 /* guard against interrupt when we're going down */
4566 if (!test_bit(__IGB_DOWN, &adapter->state))
4567 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4570 napi_schedule(&q_vector->napi);
4575 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
4577 struct igb_adapter *adapter = q_vector->adapter;
4578 struct e1000_hw *hw = &adapter->hw;
4580 if (adapter->itr_setting & 3) {
4581 if (!adapter->msix_entries)
4582 igb_set_itr(adapter);
4584 igb_update_ring_itr(q_vector);
4587 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4588 if (adapter->msix_entries)
4589 wr32(E1000_EIMS, q_vector->eims_value);
4591 igb_irq_enable(adapter);
4596 * igb_poll - NAPI Rx polling callback
4597 * @napi: napi polling structure
4598 * @budget: count of how many packets we should handle
4600 static int igb_poll(struct napi_struct *napi, int budget)
4602 struct igb_q_vector *q_vector = container_of(napi,
4603 struct igb_q_vector,
4605 int tx_clean_complete = 1, work_done = 0;
4607 #ifdef CONFIG_IGB_DCA
4608 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4609 igb_update_dca(q_vector);
4611 if (q_vector->tx_ring)
4612 tx_clean_complete = igb_clean_tx_irq(q_vector);
4614 if (q_vector->rx_ring)
4615 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4617 if (!tx_clean_complete)
4620 /* If not enough Rx work done, exit the polling mode */
4621 if (work_done < budget) {
4622 napi_complete(napi);
4623 igb_ring_irq_enable(q_vector);
4630 * igb_hwtstamp - utility function which checks for TX time stamp
4631 * @adapter: board private structure
4632 * @skb: packet that was just sent
4634 * If we were asked to do hardware stamping and such a time stamp is
4635 * available, then it must have been for this skb here because we only
4636 * allow only one such packet into the queue.
4638 static void igb_tx_hwtstamp(struct igb_adapter *adapter, struct sk_buff *skb)
4640 union skb_shared_tx *shtx = skb_tx(skb);
4641 struct e1000_hw *hw = &adapter->hw;
4643 if (unlikely(shtx->hardware)) {
4644 u32 valid = rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID;
4646 u64 regval = rd32(E1000_TXSTMPL);
4648 struct skb_shared_hwtstamps shhwtstamps;
4650 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
4651 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4652 ns = timecounter_cyc2time(&adapter->clock,
4654 timecompare_update(&adapter->compare, ns);
4655 shhwtstamps.hwtstamp = ns_to_ktime(ns);
4656 shhwtstamps.syststamp =
4657 timecompare_transform(&adapter->compare, ns);
4658 skb_tstamp_tx(skb, &shhwtstamps);
4664 * igb_clean_tx_irq - Reclaim resources after transmit completes
4665 * @q_vector: pointer to q_vector containing needed info
4666 * returns true if ring is completely cleaned
4668 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
4670 struct igb_adapter *adapter = q_vector->adapter;
4671 struct igb_ring *tx_ring = q_vector->tx_ring;
4672 struct net_device *netdev = tx_ring->netdev;
4673 struct e1000_hw *hw = &adapter->hw;
4674 struct igb_buffer *buffer_info;
4675 struct sk_buff *skb;
4676 union e1000_adv_tx_desc *tx_desc, *eop_desc;
4677 unsigned int total_bytes = 0, total_packets = 0;
4678 unsigned int i, eop, count = 0;
4679 bool cleaned = false;
4681 i = tx_ring->next_to_clean;
4682 eop = tx_ring->buffer_info[i].next_to_watch;
4683 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4685 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4686 (count < tx_ring->count)) {
4687 for (cleaned = false; !cleaned; count++) {
4688 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4689 buffer_info = &tx_ring->buffer_info[i];
4690 cleaned = (i == eop);
4691 skb = buffer_info->skb;
4694 unsigned int segs, bytecount;
4695 /* gso_segs is currently only valid for tcp */
4696 segs = skb_shinfo(skb)->gso_segs ?: 1;
4697 /* multiply data chunks by size of headers */
4698 bytecount = ((segs - 1) * skb_headlen(skb)) +
4700 total_packets += segs;
4701 total_bytes += bytecount;
4703 igb_tx_hwtstamp(adapter, skb);
4706 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4707 tx_desc->wb.status = 0;
4710 if (i == tx_ring->count)
4713 eop = tx_ring->buffer_info[i].next_to_watch;
4714 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4717 tx_ring->next_to_clean = i;
4719 if (unlikely(count &&
4720 netif_carrier_ok(netdev) &&
4721 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
4722 /* Make sure that anybody stopping the queue after this
4723 * sees the new next_to_clean.
4726 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4727 !(test_bit(__IGB_DOWN, &adapter->state))) {
4728 netif_wake_subqueue(netdev, tx_ring->queue_index);
4729 tx_ring->tx_stats.restart_queue++;
4733 if (tx_ring->detect_tx_hung) {
4734 /* Detect a transmit hang in hardware, this serializes the
4735 * check with the clearing of time_stamp and movement of i */
4736 tx_ring->detect_tx_hung = false;
4737 if (tx_ring->buffer_info[i].time_stamp &&
4738 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4739 (adapter->tx_timeout_factor * HZ))
4740 && !(rd32(E1000_STATUS) &
4741 E1000_STATUS_TXOFF)) {
4743 /* detected Tx unit hang */
4744 dev_err(&tx_ring->pdev->dev,
4745 "Detected Tx Unit Hang\n"
4749 " next_to_use <%x>\n"
4750 " next_to_clean <%x>\n"
4751 "buffer_info[next_to_clean]\n"
4752 " time_stamp <%lx>\n"
4753 " next_to_watch <%x>\n"
4755 " desc.status <%x>\n",
4756 tx_ring->queue_index,
4757 readl(tx_ring->head),
4758 readl(tx_ring->tail),
4759 tx_ring->next_to_use,
4760 tx_ring->next_to_clean,
4761 tx_ring->buffer_info[i].time_stamp,
4764 eop_desc->wb.status);
4765 netif_stop_subqueue(netdev, tx_ring->queue_index);
4768 tx_ring->total_bytes += total_bytes;
4769 tx_ring->total_packets += total_packets;
4770 tx_ring->tx_stats.bytes += total_bytes;
4771 tx_ring->tx_stats.packets += total_packets;
4772 netdev->stats.tx_bytes += total_bytes;
4773 netdev->stats.tx_packets += total_packets;
4774 return (count < tx_ring->count);
4778 * igb_receive_skb - helper function to handle rx indications
4779 * @q_vector: structure containing interrupt and ring information
4780 * @skb: packet to send up
4781 * @vlan_tag: vlan tag for packet
4783 static void igb_receive_skb(struct igb_q_vector *q_vector,
4784 struct sk_buff *skb,
4787 struct igb_adapter *adapter = q_vector->adapter;
4790 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
4793 napi_gro_receive(&q_vector->napi, skb);
4796 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
4797 u32 status_err, struct sk_buff *skb)
4799 skb->ip_summed = CHECKSUM_NONE;
4801 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
4802 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
4803 (status_err & E1000_RXD_STAT_IXSM))
4806 /* TCP/UDP checksum error bit is set */
4808 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
4810 * work around errata with sctp packets where the TCPE aka
4811 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
4812 * packets, (aka let the stack check the crc32c)
4814 if ((skb->len == 60) &&
4815 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
4816 ring->rx_stats.csum_err++;
4818 /* let the stack verify checksum errors */
4821 /* It must be a TCP or UDP packet with a valid checksum */
4822 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
4823 skb->ip_summed = CHECKSUM_UNNECESSARY;
4825 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
4828 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
4829 union e1000_adv_rx_desc *rx_desc)
4831 /* HW will not DMA in data larger than the given buffer, even if it
4832 * parses the (NFS, of course) header to be larger. In that case, it
4833 * fills the header buffer and spills the rest into the page.
4835 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
4836 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4837 if (hlen > rx_ring->rx_buffer_len)
4838 hlen = rx_ring->rx_buffer_len;
4842 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
4843 int *work_done, int budget)
4845 struct igb_adapter *adapter = q_vector->adapter;
4846 struct igb_ring *rx_ring = q_vector->rx_ring;
4847 struct net_device *netdev = rx_ring->netdev;
4848 struct e1000_hw *hw = &adapter->hw;
4849 struct pci_dev *pdev = rx_ring->pdev;
4850 union e1000_adv_rx_desc *rx_desc , *next_rxd;
4851 struct igb_buffer *buffer_info , *next_buffer;
4852 struct sk_buff *skb;
4853 bool cleaned = false;
4854 int cleaned_count = 0;
4855 unsigned int total_bytes = 0, total_packets = 0;
4861 i = rx_ring->next_to_clean;
4862 buffer_info = &rx_ring->buffer_info[i];
4863 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
4864 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
4866 while (staterr & E1000_RXD_STAT_DD) {
4867 if (*work_done >= budget)
4871 skb = buffer_info->skb;
4872 prefetch(skb->data - NET_IP_ALIGN);
4873 buffer_info->skb = NULL;
4876 if (i == rx_ring->count)
4878 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
4880 next_buffer = &rx_ring->buffer_info[i];
4882 length = le16_to_cpu(rx_desc->wb.upper.length);
4886 if (buffer_info->dma) {
4887 pci_unmap_single(pdev, buffer_info->dma,
4888 rx_ring->rx_buffer_len,
4889 PCI_DMA_FROMDEVICE);
4890 buffer_info->dma = 0;
4891 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
4892 skb_put(skb, length);
4895 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
4899 pci_unmap_page(pdev, buffer_info->page_dma,
4900 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
4901 buffer_info->page_dma = 0;
4903 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
4905 buffer_info->page_offset,
4908 if (page_count(buffer_info->page) != 1)
4909 buffer_info->page = NULL;
4911 get_page(buffer_info->page);
4914 skb->data_len += length;
4916 skb->truesize += length;
4919 if (!(staterr & E1000_RXD_STAT_EOP)) {
4920 buffer_info->skb = next_buffer->skb;
4921 buffer_info->dma = next_buffer->dma;
4922 next_buffer->skb = skb;
4923 next_buffer->dma = 0;
4928 * If this bit is set, then the RX registers contain
4929 * the time stamp. No other packet will be time
4930 * stamped until we read these registers, so read the
4931 * registers to make them available again. Because
4932 * only one packet can be time stamped at a time, we
4933 * know that the register values must belong to this
4934 * one here and therefore we don't need to compare
4935 * any of the additional attributes stored for it.
4937 * If nothing went wrong, then it should have a
4938 * skb_shared_tx that we can turn into a
4939 * skb_shared_hwtstamps.
4941 * TODO: can time stamping be triggered (thus locking
4942 * the registers) without the packet reaching this point
4943 * here? In that case RX time stamping would get stuck.
4945 * TODO: in "time stamp all packets" mode this bit is
4946 * not set. Need a global flag for this mode and then
4947 * always read the registers. Cannot be done without
4950 if (unlikely(staterr & E1000_RXD_STAT_TS)) {
4953 struct skb_shared_hwtstamps *shhwtstamps =
4956 WARN(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID),
4957 "igb: no RX time stamp available for time stamped packet");
4958 regval = rd32(E1000_RXSTMPL);
4959 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4960 ns = timecounter_cyc2time(&adapter->clock, regval);
4961 timecompare_update(&adapter->compare, ns);
4962 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
4963 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4964 shhwtstamps->syststamp =
4965 timecompare_transform(&adapter->compare, ns);
4968 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
4969 dev_kfree_skb_irq(skb);
4973 total_bytes += skb->len;
4976 igb_rx_checksum_adv(rx_ring, staterr, skb);
4978 skb->protocol = eth_type_trans(skb, netdev);
4979 skb_record_rx_queue(skb, rx_ring->queue_index);
4981 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
4982 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
4984 igb_receive_skb(q_vector, skb, vlan_tag);
4987 rx_desc->wb.upper.status_error = 0;
4989 /* return some buffers to hardware, one at a time is too slow */
4990 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
4991 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
4995 /* use prefetched values */
4997 buffer_info = next_buffer;
4998 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5001 rx_ring->next_to_clean = i;
5002 cleaned_count = igb_desc_unused(rx_ring);
5005 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5007 rx_ring->total_packets += total_packets;
5008 rx_ring->total_bytes += total_bytes;
5009 rx_ring->rx_stats.packets += total_packets;
5010 rx_ring->rx_stats.bytes += total_bytes;
5011 netdev->stats.rx_bytes += total_bytes;
5012 netdev->stats.rx_packets += total_packets;
5017 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5018 * @adapter: address of board private structure
5020 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5022 struct net_device *netdev = rx_ring->netdev;
5023 union e1000_adv_rx_desc *rx_desc;
5024 struct igb_buffer *buffer_info;
5025 struct sk_buff *skb;
5029 i = rx_ring->next_to_use;
5030 buffer_info = &rx_ring->buffer_info[i];
5032 bufsz = rx_ring->rx_buffer_len;
5034 while (cleaned_count--) {
5035 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5037 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5038 if (!buffer_info->page) {
5039 buffer_info->page = alloc_page(GFP_ATOMIC);
5040 if (!buffer_info->page) {
5041 rx_ring->rx_stats.alloc_failed++;
5044 buffer_info->page_offset = 0;
5046 buffer_info->page_offset ^= PAGE_SIZE / 2;
5048 buffer_info->page_dma =
5049 pci_map_page(rx_ring->pdev, buffer_info->page,
5050 buffer_info->page_offset,
5052 PCI_DMA_FROMDEVICE);
5055 if (!buffer_info->skb) {
5056 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5058 rx_ring->rx_stats.alloc_failed++;
5062 buffer_info->skb = skb;
5063 buffer_info->dma = pci_map_single(rx_ring->pdev,
5066 PCI_DMA_FROMDEVICE);
5068 /* Refresh the desc even if buffer_addrs didn't change because
5069 * each write-back erases this info. */
5070 if (bufsz < IGB_RXBUFFER_1024) {
5071 rx_desc->read.pkt_addr =
5072 cpu_to_le64(buffer_info->page_dma);
5073 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5075 rx_desc->read.pkt_addr =
5076 cpu_to_le64(buffer_info->dma);
5077 rx_desc->read.hdr_addr = 0;
5081 if (i == rx_ring->count)
5083 buffer_info = &rx_ring->buffer_info[i];
5087 if (rx_ring->next_to_use != i) {
5088 rx_ring->next_to_use = i;
5090 i = (rx_ring->count - 1);
5094 /* Force memory writes to complete before letting h/w
5095 * know there are new descriptors to fetch. (Only
5096 * applicable for weak-ordered memory model archs,
5097 * such as IA-64). */
5099 writel(i, rx_ring->tail);
5109 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5111 struct igb_adapter *adapter = netdev_priv(netdev);
5112 struct mii_ioctl_data *data = if_mii(ifr);
5114 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5119 data->phy_id = adapter->hw.phy.addr;
5122 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5134 * igb_hwtstamp_ioctl - control hardware time stamping
5139 * Outgoing time stamping can be enabled and disabled. Play nice and
5140 * disable it when requested, although it shouldn't case any overhead
5141 * when no packet needs it. At most one packet in the queue may be
5142 * marked for time stamping, otherwise it would be impossible to tell
5143 * for sure to which packet the hardware time stamp belongs.
5145 * Incoming time stamping has to be configured via the hardware
5146 * filters. Not all combinations are supported, in particular event
5147 * type has to be specified. Matching the kind of event packet is
5148 * not supported, with the exception of "all V2 events regardless of
5152 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5153 struct ifreq *ifr, int cmd)
5155 struct igb_adapter *adapter = netdev_priv(netdev);
5156 struct e1000_hw *hw = &adapter->hw;
5157 struct hwtstamp_config config;
5158 u32 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
5159 u32 tsync_rx_ctl_bit = E1000_TSYNCRXCTL_ENABLED;
5160 u32 tsync_rx_ctl_type = 0;
5161 u32 tsync_rx_cfg = 0;
5164 short port = 319; /* PTP */
5167 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5170 /* reserved for future extensions */
5174 switch (config.tx_type) {
5175 case HWTSTAMP_TX_OFF:
5176 tsync_tx_ctl_bit = 0;
5178 case HWTSTAMP_TX_ON:
5179 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
5185 switch (config.rx_filter) {
5186 case HWTSTAMP_FILTER_NONE:
5187 tsync_rx_ctl_bit = 0;
5189 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5190 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5191 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5192 case HWTSTAMP_FILTER_ALL:
5194 * register TSYNCRXCFG must be set, therefore it is not
5195 * possible to time stamp both Sync and Delay_Req messages
5196 * => fall back to time stamping all packets
5198 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_ALL;
5199 config.rx_filter = HWTSTAMP_FILTER_ALL;
5201 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5202 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
5203 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5206 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5207 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
5208 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5211 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5212 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5213 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5214 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5217 config.rx_filter = HWTSTAMP_FILTER_SOME;
5219 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5220 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5221 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5222 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5225 config.rx_filter = HWTSTAMP_FILTER_SOME;
5227 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5228 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5229 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5230 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5231 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5238 /* enable/disable TX */
5239 regval = rd32(E1000_TSYNCTXCTL);
5240 regval = (regval & ~E1000_TSYNCTXCTL_ENABLED) | tsync_tx_ctl_bit;
5241 wr32(E1000_TSYNCTXCTL, regval);
5243 /* enable/disable RX, define which PTP packets are time stamped */
5244 regval = rd32(E1000_TSYNCRXCTL);
5245 regval = (regval & ~E1000_TSYNCRXCTL_ENABLED) | tsync_rx_ctl_bit;
5246 regval = (regval & ~0xE) | tsync_rx_ctl_type;
5247 wr32(E1000_TSYNCRXCTL, regval);
5248 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5251 * Ethertype Filter Queue Filter[0][15:0] = 0x88F7
5252 * (Ethertype to filter on)
5253 * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter)
5254 * Ethertype Filter Queue Filter[0][30] = 0x1 (Enable Timestamping)
5256 wr32(E1000_ETQF0, is_l2 ? 0x440088f7 : 0);
5258 /* L4 Queue Filter[0]: only filter by source and destination port */
5259 wr32(E1000_SPQF0, htons(port));
5260 wr32(E1000_IMIREXT(0), is_l4 ?
5261 ((1<<12) | (1<<19) /* bypass size and control flags */) : 0);
5262 wr32(E1000_IMIR(0), is_l4 ?
5264 | (0<<16) /* immediate interrupt disabled */
5265 | 0 /* (1<<17) bit cleared: do not bypass
5266 destination port check */)
5268 wr32(E1000_FTQF0, is_l4 ?
5270 | (1<<15) /* VF not compared */
5271 | (1<<27) /* Enable Timestamping */
5272 | (7<<28) /* only source port filter enabled,
5273 source/target address and protocol
5275 : ((1<<15) | (15<<28) /* all mask bits set = filter not
5280 adapter->hwtstamp_config = config;
5282 /* clear TX/RX time stamp registers, just to be sure */
5283 regval = rd32(E1000_TXSTMPH);
5284 regval = rd32(E1000_RXSTMPH);
5286 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5296 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5302 return igb_mii_ioctl(netdev, ifr, cmd);
5304 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5310 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5312 struct igb_adapter *adapter = hw->back;
5315 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5317 return -E1000_ERR_CONFIG;
5319 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5324 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5326 struct igb_adapter *adapter = hw->back;
5329 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5331 return -E1000_ERR_CONFIG;
5333 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5338 static void igb_vlan_rx_register(struct net_device *netdev,
5339 struct vlan_group *grp)
5341 struct igb_adapter *adapter = netdev_priv(netdev);
5342 struct e1000_hw *hw = &adapter->hw;
5345 igb_irq_disable(adapter);
5346 adapter->vlgrp = grp;
5349 /* enable VLAN tag insert/strip */
5350 ctrl = rd32(E1000_CTRL);
5351 ctrl |= E1000_CTRL_VME;
5352 wr32(E1000_CTRL, ctrl);
5354 /* enable VLAN receive filtering */
5355 rctl = rd32(E1000_RCTL);
5356 rctl &= ~E1000_RCTL_CFIEN;
5357 wr32(E1000_RCTL, rctl);
5358 igb_update_mng_vlan(adapter);
5360 /* disable VLAN tag insert/strip */
5361 ctrl = rd32(E1000_CTRL);
5362 ctrl &= ~E1000_CTRL_VME;
5363 wr32(E1000_CTRL, ctrl);
5365 if (adapter->mng_vlan_id != (u16)IGB_MNG_VLAN_NONE) {
5366 igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
5367 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
5371 igb_rlpml_set(adapter);
5373 if (!test_bit(__IGB_DOWN, &adapter->state))
5374 igb_irq_enable(adapter);
5377 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5379 struct igb_adapter *adapter = netdev_priv(netdev);
5380 struct e1000_hw *hw = &adapter->hw;
5381 int pf_id = adapter->vfs_allocated_count;
5383 if ((hw->mng_cookie.status &
5384 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
5385 (vid == adapter->mng_vlan_id))
5388 /* add vid to vlvf if sr-iov is enabled,
5389 * if that fails add directly to filter table */
5390 if (igb_vlvf_set(adapter, vid, true, pf_id))
5391 igb_vfta_set(hw, vid, true);
5395 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5397 struct igb_adapter *adapter = netdev_priv(netdev);
5398 struct e1000_hw *hw = &adapter->hw;
5399 int pf_id = adapter->vfs_allocated_count;
5401 igb_irq_disable(adapter);
5402 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5404 if (!test_bit(__IGB_DOWN, &adapter->state))
5405 igb_irq_enable(adapter);
5407 if ((adapter->hw.mng_cookie.status &
5408 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
5409 (vid == adapter->mng_vlan_id)) {
5410 /* release control to f/w */
5411 igb_release_hw_control(adapter);
5415 /* remove vid from vlvf if sr-iov is enabled,
5416 * if not in vlvf remove from vfta */
5417 if (igb_vlvf_set(adapter, vid, false, pf_id))
5418 igb_vfta_set(hw, vid, false);
5421 static void igb_restore_vlan(struct igb_adapter *adapter)
5423 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5425 if (adapter->vlgrp) {
5427 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5428 if (!vlan_group_get_device(adapter->vlgrp, vid))
5430 igb_vlan_rx_add_vid(adapter->netdev, vid);
5435 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5437 struct e1000_mac_info *mac = &adapter->hw.mac;
5442 case SPEED_10 + DUPLEX_HALF:
5443 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5445 case SPEED_10 + DUPLEX_FULL:
5446 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5448 case SPEED_100 + DUPLEX_HALF:
5449 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5451 case SPEED_100 + DUPLEX_FULL:
5452 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5454 case SPEED_1000 + DUPLEX_FULL:
5456 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5458 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5460 dev_err(&adapter->pdev->dev,
5461 "Unsupported Speed/Duplex configuration\n");
5467 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
5469 struct net_device *netdev = pci_get_drvdata(pdev);
5470 struct igb_adapter *adapter = netdev_priv(netdev);
5471 struct e1000_hw *hw = &adapter->hw;
5472 u32 ctrl, rctl, status;
5473 u32 wufc = adapter->wol;
5478 netif_device_detach(netdev);
5480 if (netif_running(netdev))
5483 igb_clear_interrupt_scheme(adapter);
5486 retval = pci_save_state(pdev);
5491 status = rd32(E1000_STATUS);
5492 if (status & E1000_STATUS_LU)
5493 wufc &= ~E1000_WUFC_LNKC;
5496 igb_setup_rctl(adapter);
5497 igb_set_rx_mode(netdev);
5499 /* turn on all-multi mode if wake on multicast is enabled */
5500 if (wufc & E1000_WUFC_MC) {
5501 rctl = rd32(E1000_RCTL);
5502 rctl |= E1000_RCTL_MPE;
5503 wr32(E1000_RCTL, rctl);
5506 ctrl = rd32(E1000_CTRL);
5507 /* advertise wake from D3Cold */
5508 #define E1000_CTRL_ADVD3WUC 0x00100000
5509 /* phy power management enable */
5510 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5511 ctrl |= E1000_CTRL_ADVD3WUC;
5512 wr32(E1000_CTRL, ctrl);
5514 /* Allow time for pending master requests to run */
5515 igb_disable_pcie_master(&adapter->hw);
5517 wr32(E1000_WUC, E1000_WUC_PME_EN);
5518 wr32(E1000_WUFC, wufc);
5521 wr32(E1000_WUFC, 0);
5524 *enable_wake = wufc || adapter->en_mng_pt;
5526 igb_shutdown_serdes_link_82575(hw);
5528 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5529 * would have already happened in close and is redundant. */
5530 igb_release_hw_control(adapter);
5532 pci_disable_device(pdev);
5538 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5543 retval = __igb_shutdown(pdev, &wake);
5548 pci_prepare_to_sleep(pdev);
5550 pci_wake_from_d3(pdev, false);
5551 pci_set_power_state(pdev, PCI_D3hot);
5557 static int igb_resume(struct pci_dev *pdev)
5559 struct net_device *netdev = pci_get_drvdata(pdev);
5560 struct igb_adapter *adapter = netdev_priv(netdev);
5561 struct e1000_hw *hw = &adapter->hw;
5564 pci_set_power_state(pdev, PCI_D0);
5565 pci_restore_state(pdev);
5567 err = pci_enable_device_mem(pdev);
5570 "igb: Cannot enable PCI device from suspend\n");
5573 pci_set_master(pdev);
5575 pci_enable_wake(pdev, PCI_D3hot, 0);
5576 pci_enable_wake(pdev, PCI_D3cold, 0);
5578 if (igb_init_interrupt_scheme(adapter)) {
5579 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5583 /* e1000_power_up_phy(adapter); */
5587 /* let the f/w know that the h/w is now under the control of the
5589 igb_get_hw_control(adapter);
5591 wr32(E1000_WUS, ~0);
5593 if (netif_running(netdev)) {
5594 err = igb_open(netdev);
5599 netif_device_attach(netdev);
5605 static void igb_shutdown(struct pci_dev *pdev)
5609 __igb_shutdown(pdev, &wake);
5611 if (system_state == SYSTEM_POWER_OFF) {
5612 pci_wake_from_d3(pdev, wake);
5613 pci_set_power_state(pdev, PCI_D3hot);
5617 #ifdef CONFIG_NET_POLL_CONTROLLER
5619 * Polling 'interrupt' - used by things like netconsole to send skbs
5620 * without having to re-enable interrupts. It's not called while
5621 * the interrupt routine is executing.
5623 static void igb_netpoll(struct net_device *netdev)
5625 struct igb_adapter *adapter = netdev_priv(netdev);
5626 struct e1000_hw *hw = &adapter->hw;
5629 if (!adapter->msix_entries) {
5630 struct igb_q_vector *q_vector = adapter->q_vector[0];
5631 igb_irq_disable(adapter);
5632 napi_schedule(&q_vector->napi);
5636 for (i = 0; i < adapter->num_q_vectors; i++) {
5637 struct igb_q_vector *q_vector = adapter->q_vector[i];
5638 wr32(E1000_EIMC, q_vector->eims_value);
5639 napi_schedule(&q_vector->napi);
5642 #endif /* CONFIG_NET_POLL_CONTROLLER */
5645 * igb_io_error_detected - called when PCI error is detected
5646 * @pdev: Pointer to PCI device
5647 * @state: The current pci connection state
5649 * This function is called after a PCI bus error affecting
5650 * this device has been detected.
5652 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5653 pci_channel_state_t state)
5655 struct net_device *netdev = pci_get_drvdata(pdev);
5656 struct igb_adapter *adapter = netdev_priv(netdev);
5658 netif_device_detach(netdev);
5660 if (state == pci_channel_io_perm_failure)
5661 return PCI_ERS_RESULT_DISCONNECT;
5663 if (netif_running(netdev))
5665 pci_disable_device(pdev);
5667 /* Request a slot slot reset. */
5668 return PCI_ERS_RESULT_NEED_RESET;
5672 * igb_io_slot_reset - called after the pci bus has been reset.
5673 * @pdev: Pointer to PCI device
5675 * Restart the card from scratch, as if from a cold-boot. Implementation
5676 * resembles the first-half of the igb_resume routine.
5678 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5680 struct net_device *netdev = pci_get_drvdata(pdev);
5681 struct igb_adapter *adapter = netdev_priv(netdev);
5682 struct e1000_hw *hw = &adapter->hw;
5683 pci_ers_result_t result;
5686 if (pci_enable_device_mem(pdev)) {
5688 "Cannot re-enable PCI device after reset.\n");
5689 result = PCI_ERS_RESULT_DISCONNECT;
5691 pci_set_master(pdev);
5692 pci_restore_state(pdev);
5694 pci_enable_wake(pdev, PCI_D3hot, 0);
5695 pci_enable_wake(pdev, PCI_D3cold, 0);
5698 wr32(E1000_WUS, ~0);
5699 result = PCI_ERS_RESULT_RECOVERED;
5702 err = pci_cleanup_aer_uncorrect_error_status(pdev);
5704 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5705 "failed 0x%0x\n", err);
5706 /* non-fatal, continue */
5713 * igb_io_resume - called when traffic can start flowing again.
5714 * @pdev: Pointer to PCI device
5716 * This callback is called when the error recovery driver tells us that
5717 * its OK to resume normal operation. Implementation resembles the
5718 * second-half of the igb_resume routine.
5720 static void igb_io_resume(struct pci_dev *pdev)
5722 struct net_device *netdev = pci_get_drvdata(pdev);
5723 struct igb_adapter *adapter = netdev_priv(netdev);
5725 if (netif_running(netdev)) {
5726 if (igb_up(adapter)) {
5727 dev_err(&pdev->dev, "igb_up failed after reset\n");
5732 netif_device_attach(netdev);
5734 /* let the f/w know that the h/w is now under the control of the
5736 igb_get_hw_control(adapter);
5739 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5742 u32 rar_low, rar_high;
5743 struct e1000_hw *hw = &adapter->hw;
5745 /* HW expects these in little endian so we reverse the byte order
5746 * from network order (big endian) to little endian
5748 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5749 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5750 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5752 /* Indicate to hardware the Address is Valid. */
5753 rar_high |= E1000_RAH_AV;
5755 if (hw->mac.type == e1000_82575)
5756 rar_high |= E1000_RAH_POOL_1 * qsel;
5758 rar_high |= E1000_RAH_POOL_1 << qsel;
5760 wr32(E1000_RAL(index), rar_low);
5762 wr32(E1000_RAH(index), rar_high);
5766 static int igb_set_vf_mac(struct igb_adapter *adapter,
5767 int vf, unsigned char *mac_addr)
5769 struct e1000_hw *hw = &adapter->hw;
5770 /* VF MAC addresses start at end of receive addresses and moves
5771 * torwards the first, as a result a collision should not be possible */
5772 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5774 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
5776 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
5781 static void igb_vmm_control(struct igb_adapter *adapter)
5783 struct e1000_hw *hw = &adapter->hw;
5786 /* replication is not supported for 82575 */
5787 if (hw->mac.type == e1000_82575)
5790 /* enable replication vlan tag stripping */
5791 reg = rd32(E1000_RPLOLR);
5792 reg |= E1000_RPLOLR_STRVLAN;
5793 wr32(E1000_RPLOLR, reg);
5795 /* notify HW that the MAC is adding vlan tags */
5796 reg = rd32(E1000_DTXCTL);
5797 reg |= E1000_DTXCTL_VLAN_ADDED;
5798 wr32(E1000_DTXCTL, reg);
5800 if (adapter->vfs_allocated_count) {
5801 igb_vmdq_set_loopback_pf(hw, true);
5802 igb_vmdq_set_replication_pf(hw, true);
5804 igb_vmdq_set_loopback_pf(hw, false);
5805 igb_vmdq_set_replication_pf(hw, false);