SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/net/tg3.c

   1 /*
   2  * tg3.c: Broadcom Tigon3 ethernet driver.
   3  *
   4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
   5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
   6  * Copyright (C) 2004 Sun Microsystems Inc.
   7  * Copyright (C) 2005-2007 Broadcom Corporation.
   8  *
   9  * Firmware is:
  10  *      Derived from proprietary unpublished source code,
  11  *      Copyright (C) 2000-2003 Broadcom Corporation.
  12  *
  13  *      Permission is hereby granted for the distribution of this firmware
  14  *      data in hexadecimal or equivalent format, provided this copyright
  15  *      notice is accompanying it.
  16  */
  17
  18
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kernel.h>
  22 #include <linux/types.h>
  23 #include <linux/compiler.h>
  24 #include <linux/slab.h>
  25 #include <linux/delay.h>
  26 #include <linux/in.h>
  27 #include <linux/init.h>
  28 #include <linux/ioport.h>
  29 #include <linux/pci.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/mii.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/workqueue.h>
  39 #include <linux/prefetch.h>
  40 #include <linux/dma-mapping.h>
  41
  42 #include <net/checksum.h>
  43 #include <net/ip.h>
  44
  45 #include <asm/system.h>
  46 #include <asm/io.h>
  47 #include <asm/byteorder.h>
  48 #include <asm/uaccess.h>
  49
  50 #ifdef CONFIG_SPARC
  51 #include <asm/idprom.h>
  52 #include <asm/prom.h>
  53 #endif
  54
  55 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
  56 #define TG3_VLAN_TAG_USED 1
  57 #else
  58 #define TG3_VLAN_TAG_USED 0
  59 #endif
  60
  61 #define TG3_TSO_SUPPORT 1
  62
  63 #include "tg3.h"
  64
  65 #define DRV_MODULE_NAME         "tg3"
  66 #define PFX DRV_MODULE_NAME     ": "
  67 #define DRV_MODULE_VERSION      "3.85"
  68 #define DRV_MODULE_RELDATE      "October 18, 2007"
  69
  70 #define TG3_DEF_MAC_MODE        0
  71 #define TG3_DEF_RX_MODE         0
  72 #define TG3_DEF_TX_MODE         0
  73 #define TG3_DEF_MSG_ENABLE        \
  74         (NETIF_MSG_DRV          | \
  75          NETIF_MSG_PROBE        | \
  76          NETIF_MSG_LINK         | \
  77          NETIF_MSG_TIMER        | \
  78          NETIF_MSG_IFDOWN       | \
  79          NETIF_MSG_IFUP         | \
  80          NETIF_MSG_RX_ERR       | \
  81          NETIF_MSG_TX_ERR)
  82
  83 /* length of time before we decide the hardware is borked,
  84  * and dev->tx_timeout() should be called to fix the problem
  85  */
  86 #define TG3_TX_TIMEOUT                  (5 * HZ)
  87
  88 /* hardware minimum and maximum for a single frame's data payload */
  89 #define TG3_MIN_MTU                     60
  90 #define TG3_MAX_MTU(tp) \
  91         ((tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) ? 9000 : 1500)
  92
  93 /* These numbers seem to be hard coded in the NIC firmware somehow.
  94  * You can't change the ring sizes, but you can change where you place
  95  * them in the NIC onboard memory.
  96  */
  97 #define TG3_RX_RING_SIZE                512
  98 #define TG3_DEF_RX_RING_PENDING         200
  99 #define TG3_RX_JUMBO_RING_SIZE          256
 100 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
 101
 102 /* Do not place this n-ring entries value into the tp struct itself,
 103  * we really want to expose these constants to GCC so that modulo et
 104  * al.  operations are done with shifts and masks instead of with
 105  * hw multiply/modulo instructions.  Another solution would be to
 106  * replace things like '% foo' with '& (foo - 1)'.
 107  */
 108 #define TG3_RX_RCB_RING_SIZE(tp)        \
 109         ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ?  512 : 1024)
 110
 111 #define TG3_TX_RING_SIZE                512
 112 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
 113
 114 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
 115                                  TG3_RX_RING_SIZE)
 116 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_rx_buffer_desc) * \
 117                                  TG3_RX_JUMBO_RING_SIZE)
 118 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
 119                                    TG3_RX_RCB_RING_SIZE(tp))
 120 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
 121                                  TG3_TX_RING_SIZE)
 122 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
 123
 124 #define RX_PKT_BUF_SZ           (1536 + tp->rx_offset + 64)
 125 #define RX_JUMBO_PKT_BUF_SZ     (9046 + tp->rx_offset + 64)
 126
 127 /* minimum number of free TX descriptors required to wake up TX process */
 128 #define TG3_TX_WAKEUP_THRESH(tp)                ((tp)->tx_pending / 4)
 129
 130 /* number of ETHTOOL_GSTATS u64's */
 131 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
 132
 133 #define TG3_NUM_TEST            6
 134
 135 static char version[] __devinitdata =
 136         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 137
 138 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
 139 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
 140 MODULE_LICENSE("GPL");
 141 MODULE_VERSION(DRV_MODULE_VERSION);
 142
 143 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
 144 module_param(tg3_debug, int, 0);
 145 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 146
 147 static struct pci_device_id tg3_pci_tbl[] = {
 148         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
 149         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
 150         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
 151         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
 152         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
 153         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
 154         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
 155         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
 156         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
 157         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
 158         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
 159         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
 160         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
 161         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
 162         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
 163         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
 164         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
 165         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
 166         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
 167         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
 168         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
 169         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
 170         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720)},
 171         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
 172         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
 173         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
 174         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
 175         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M)},
 176         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
 177         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
 178         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
 179         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
 180         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
 181         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
 182         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
 183         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
 184         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
 185         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
 186         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
 187         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
 188         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
 189         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
 190         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
 191         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
 192         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
 193         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
 194         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
 195         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
 196         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
 197         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
 198         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
 199         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
 200         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
 201         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
 202         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
 203         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
 204         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
 205         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
 206         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
 207         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
 208         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
 209         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
 210         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
 211         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
 212         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
 213         {}
 214 };
 215
 216 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 217
 218 static const struct {
 219         const char string[ETH_GSTRING_LEN];
 220 } ethtool_stats_keys[TG3_NUM_STATS] = {
 221         { "rx_octets" },
 222         { "rx_fragments" },
 223         { "rx_ucast_packets" },
 224         { "rx_mcast_packets" },
 225         { "rx_bcast_packets" },
 226         { "rx_fcs_errors" },
 227         { "rx_align_errors" },
 228         { "rx_xon_pause_rcvd" },
 229         { "rx_xoff_pause_rcvd" },
 230         { "rx_mac_ctrl_rcvd" },
 231         { "rx_xoff_entered" },
 232         { "rx_frame_too_long_errors" },
 233         { "rx_jabbers" },
 234         { "rx_undersize_packets" },
 235         { "rx_in_length_errors" },
 236         { "rx_out_length_errors" },
 237         { "rx_64_or_less_octet_packets" },
 238         { "rx_65_to_127_octet_packets" },
 239         { "rx_128_to_255_octet_packets" },
 240         { "rx_256_to_511_octet_packets" },
 241         { "rx_512_to_1023_octet_packets" },
 242         { "rx_1024_to_1522_octet_packets" },
 243         { "rx_1523_to_2047_octet_packets" },
 244         { "rx_2048_to_4095_octet_packets" },
 245         { "rx_4096_to_8191_octet_packets" },
 246         { "rx_8192_to_9022_octet_packets" },
 247
 248         { "tx_octets" },
 249         { "tx_collisions" },
 250
 251         { "tx_xon_sent" },
 252         { "tx_xoff_sent" },
 253         { "tx_flow_control" },
 254         { "tx_mac_errors" },
 255         { "tx_single_collisions" },
 256         { "tx_mult_collisions" },
 257         { "tx_deferred" },
 258         { "tx_excessive_collisions" },
 259         { "tx_late_collisions" },
 260         { "tx_collide_2times" },
 261         { "tx_collide_3times" },
 262         { "tx_collide_4times" },
 263         { "tx_collide_5times" },
 264         { "tx_collide_6times" },
 265         { "tx_collide_7times" },
 266         { "tx_collide_8times" },
 267         { "tx_collide_9times" },
 268         { "tx_collide_10times" },
 269         { "tx_collide_11times" },
 270         { "tx_collide_12times" },
 271         { "tx_collide_13times" },
 272         { "tx_collide_14times" },
 273         { "tx_collide_15times" },
 274         { "tx_ucast_packets" },
 275         { "tx_mcast_packets" },
 276         { "tx_bcast_packets" },
 277         { "tx_carrier_sense_errors" },
 278         { "tx_discards" },
 279         { "tx_errors" },
 280
 281         { "dma_writeq_full" },
 282         { "dma_write_prioq_full" },
 283         { "rxbds_empty" },
 284         { "rx_discards" },
 285         { "rx_errors" },
 286         { "rx_threshold_hit" },
 287
 288         { "dma_readq_full" },
 289         { "dma_read_prioq_full" },
 290         { "tx_comp_queue_full" },
 291
 292         { "ring_set_send_prod_index" },
 293         { "ring_status_update" },
 294         { "nic_irqs" },
 295         { "nic_avoided_irqs" },
 296         { "nic_tx_threshold_hit" }
 297 };
 298
 299 static const struct {
 300         const char string[ETH_GSTRING_LEN];
 301 } ethtool_test_keys[TG3_NUM_TEST] = {
 302         { "nvram test     (online) " },
 303         { "link test      (online) " },
 304         { "register test  (offline)" },
 305         { "memory test    (offline)" },
 306         { "loopback test  (offline)" },
 307         { "interrupt test (offline)" },
 308 };
 309
 310 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
 311 {
 312         writel(val, tp->regs + off);
 313 }
 314
 315 static u32 tg3_read32(struct tg3 *tp, u32 off)
 316 {
 317         return (readl(tp->regs + off));
 318 }
 319
 320 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
 321 {
 322         writel(val, tp->aperegs + off);
 323 }
 324
 325 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
 326 {
 327         return (readl(tp->aperegs + off));
 328 }
 329
 330 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
 331 {
 332         unsigned long flags;
 333
 334         spin_lock_irqsave(&tp->indirect_lock, flags);
 335         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 336         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 337         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 338 }
 339
 340 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
 341 {
 342         writel(val, tp->regs + off);
 343         readl(tp->regs + off);
 344 }
 345
 346 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
 347 {
 348         unsigned long flags;
 349         u32 val;
 350
 351         spin_lock_irqsave(&tp->indirect_lock, flags);
 352         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 353         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 354         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 355         return val;
 356 }
 357
 358 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
 359 {
 360         unsigned long flags;
 361
 362         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
 363                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
 364                                        TG3_64BIT_REG_LOW, val);
 365                 return;
 366         }
 367         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
 368                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
 369                                        TG3_64BIT_REG_LOW, val);
 370                 return;
 371         }
 372
 373         spin_lock_irqsave(&tp->indirect_lock, flags);
 374         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 375         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 376         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 377
 378         /* In indirect mode when disabling interrupts, we also need
 379          * to clear the interrupt bit in the GRC local ctrl register.
 380          */
 381         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
 382             (val == 0x1)) {
 383                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
 384                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
 385         }
 386 }
 387
 388 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
 389 {
 390         unsigned long flags;
 391         u32 val;
 392
 393         spin_lock_irqsave(&tp->indirect_lock, flags);
 394         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 395         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 396         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 397         return val;
 398 }
 399
 400 /* usec_wait specifies the wait time in usec when writing to certain registers
 401  * where it is unsafe to read back the register without some delay.
 402  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
 403  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
 404  */
 405 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
 406 {
 407         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
 408             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 409                 /* Non-posted methods */
 410                 tp->write32(tp, off, val);
 411         else {
 412                 /* Posted method */
 413                 tg3_write32(tp, off, val);
 414                 if (usec_wait)
 415                         udelay(usec_wait);
 416                 tp->read32(tp, off);
 417         }
 418         /* Wait again after the read for the posted method to guarantee that
 419          * the wait time is met.
 420          */
 421         if (usec_wait)
 422                 udelay(usec_wait);
 423 }
 424
 425 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
 426 {
 427         tp->write32_mbox(tp, off, val);
 428         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
 429             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 430                 tp->read32_mbox(tp, off);
 431 }
 432
 433 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
 434 {
 435         void __iomem *mbox = tp->regs + off;
 436         writel(val, mbox);
 437         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
 438                 writel(val, mbox);
 439         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
 440                 readl(mbox);
 441 }
 442
 443 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
 444 {
 445         return (readl(tp->regs + off + GRCMBOX_BASE));
 446 }
 447
 448 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
 449 {
 450         writel(val, tp->regs + off + GRCMBOX_BASE);
 451 }
 452
 453 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
 454 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
 455 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
 456 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
 457 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
 458
 459 #define tw32(reg,val)           tp->write32(tp, reg, val)
 460 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
 461 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
 462 #define tr32(reg)               tp->read32(tp, reg)
 463
 464 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
 465 {
 466         unsigned long flags;
 467
 468         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 469             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
 470                 return;
 471
 472         spin_lock_irqsave(&tp->indirect_lock, flags);
 473         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 474                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 475                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 476
 477                 /* Always leave this as zero. */
 478                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 479         } else {
 480                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 481                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
 482
 483                 /* Always leave this as zero. */
 484                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 485         }
 486         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 487 }
 488
 489 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
 490 {
 491         unsigned long flags;
 492
 493         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 494             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
 495                 *val = 0;
 496                 return;
 497         }
 498
 499         spin_lock_irqsave(&tp->indirect_lock, flags);
 500         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 501                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 502                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 503
 504                 /* Always leave this as zero. */
 505                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 506         } else {
 507                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 508                 *val = tr32(TG3PCI_MEM_WIN_DATA);
 509
 510                 /* Always leave this as zero. */
 511                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 512         }
 513         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 514 }
 515
 516 static void tg3_ape_lock_init(struct tg3 *tp)
 517 {
 518         int i;
 519
 520         /* Make sure the driver hasn't any stale locks. */
 521         for (i = 0; i < 8; i++)
 522                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + 4 * i,
 523                                 APE_LOCK_GRANT_DRIVER);
 524 }
 525
 526 static int tg3_ape_lock(struct tg3 *tp, int locknum)
 527 {
 528         int i, off;
 529         int ret = 0;
 530         u32 status;
 531
 532         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
 533                 return 0;
 534
 535         switch (locknum) {
 536                 case TG3_APE_LOCK_MEM:
 537                         break;
 538                 default:
 539                         return -EINVAL;
 540         }
 541
 542         off = 4 * locknum;
 543
 544         tg3_ape_write32(tp, TG3_APE_LOCK_REQ + off, APE_LOCK_REQ_DRIVER);
 545
 546         /* Wait for up to 1 millisecond to acquire lock. */
 547         for (i = 0; i < 100; i++) {
 548                 status = tg3_ape_read32(tp, TG3_APE_LOCK_GRANT + off);
 549                 if (status == APE_LOCK_GRANT_DRIVER)
 550                         break;
 551                 udelay(10);
 552         }
 553
 554         if (status != APE_LOCK_GRANT_DRIVER) {
 555                 /* Revoke the lock request. */
 556                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off,
 557                                 APE_LOCK_GRANT_DRIVER);
 558
 559                 ret = -EBUSY;
 560         }
 561
 562         return ret;
 563 }
 564
 565 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
 566 {
 567         int off;
 568
 569         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
 570                 return;
 571
 572         switch (locknum) {
 573                 case TG3_APE_LOCK_MEM:
 574                         break;
 575                 default:
 576                         return;
 577         }
 578
 579         off = 4 * locknum;
 580         tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off, APE_LOCK_GRANT_DRIVER);
 581 }
 582
 583 static void tg3_disable_ints(struct tg3 *tp)
 584 {
 585         tw32(TG3PCI_MISC_HOST_CTRL,
 586              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
 587         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
 588 }
 589
 590 static inline void tg3_cond_int(struct tg3 *tp)
 591 {
 592         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 593             (tp->hw_status->status & SD_STATUS_UPDATED))
 594                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
 595         else
 596                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 597                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 598 }
 599
 600 static void tg3_enable_ints(struct tg3 *tp)
 601 {
 602         tp->irq_sync = 0;
 603         wmb();
 604
 605         tw32(TG3PCI_MISC_HOST_CTRL,
 606              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
 607         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 608                        (tp->last_tag << 24));
 609         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
 610                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 611                                (tp->last_tag << 24));
 612         tg3_cond_int(tp);
 613 }
 614
 615 static inline unsigned int tg3_has_work(struct tg3 *tp)
 616 {
 617         struct tg3_hw_status *sblk = tp->hw_status;
 618         unsigned int work_exists = 0;
 619
 620         /* check for phy events */
 621         if (!(tp->tg3_flags &
 622               (TG3_FLAG_USE_LINKCHG_REG |
 623                TG3_FLAG_POLL_SERDES))) {
 624                 if (sblk->status & SD_STATUS_LINK_CHG)
 625                         work_exists = 1;
 626         }
 627         /* check for RX/TX work to do */
 628         if (sblk->idx[0].tx_consumer != tp->tx_cons ||
 629             sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
 630                 work_exists = 1;
 631
 632         return work_exists;
 633 }
 634
 635 /* tg3_restart_ints
 636  *  similar to tg3_enable_ints, but it accurately determines whether there
 637  *  is new work pending and can return without flushing the PIO write
 638  *  which reenables interrupts
 639  */
 640 static void tg3_restart_ints(struct tg3 *tp)
 641 {
 642         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 643                      tp->last_tag << 24);
 644         mmiowb();
 645
 646         /* When doing tagged status, this work check is unnecessary.
 647          * The last_tag we write above tells the chip which piece of
 648          * work we've completed.
 649          */
 650         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 651             tg3_has_work(tp))
 652                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 653                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 654 }
 655
 656 static inline void tg3_netif_stop(struct tg3 *tp)
 657 {
 658         tp->dev->trans_start = jiffies; /* prevent tx timeout */
 659         napi_disable(&tp->napi);
 660         netif_tx_disable(tp->dev);
 661 }
 662
 663 static inline void tg3_netif_start(struct tg3 *tp)
 664 {
 665         netif_wake_queue(tp->dev);
 666         /* NOTE: unconditional netif_wake_queue is only appropriate
 667          * so long as all callers are assured to have free tx slots
 668          * (such as after tg3_init_hw)
 669          */
 670         napi_enable(&tp->napi);
 671         tp->hw_status->status |= SD_STATUS_UPDATED;
 672         tg3_enable_ints(tp);
 673 }
 674
 675 static void tg3_switch_clocks(struct tg3 *tp)
 676 {
 677         u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
 678         u32 orig_clock_ctrl;
 679
 680         if ((tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
 681             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
 682                 return;
 683
 684         orig_clock_ctrl = clock_ctrl;
 685         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
 686                        CLOCK_CTRL_CLKRUN_OENABLE |
 687                        0x1f);
 688         tp->pci_clock_ctrl = clock_ctrl;
 689
 690         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
 691                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
 692                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
 693                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
 694                 }
 695         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 696                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 697                             clock_ctrl |
 698                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
 699                             40);
 700                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 701                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
 702                             40);
 703         }
 704         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
 705 }
 706
 707 #define PHY_BUSY_LOOPS  5000
 708
 709 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
 710 {
 711         u32 frame_val;
 712         unsigned int loops;
 713         int ret;
 714
 715         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 716                 tw32_f(MAC_MI_MODE,
 717                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 718                 udelay(80);
 719         }
 720
 721         *val = 0x0;
 722
 723         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 724                       MI_COM_PHY_ADDR_MASK);
 725         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 726                       MI_COM_REG_ADDR_MASK);
 727         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
 728
 729         tw32_f(MAC_MI_COM, frame_val);
 730
 731         loops = PHY_BUSY_LOOPS;
 732         while (loops != 0) {
 733                 udelay(10);
 734                 frame_val = tr32(MAC_MI_COM);
 735
 736                 if ((frame_val & MI_COM_BUSY) == 0) {
 737                         udelay(5);
 738                         frame_val = tr32(MAC_MI_COM);
 739                         break;
 740                 }
 741                 loops -= 1;
 742         }
 743
 744         ret = -EBUSY;
 745         if (loops != 0) {
 746                 *val = frame_val & MI_COM_DATA_MASK;
 747                 ret = 0;
 748         }
 749
 750         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 751                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 752                 udelay(80);
 753         }
 754
 755         return ret;
 756 }
 757
 758 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
 759 {
 760         u32 frame_val;
 761         unsigned int loops;
 762         int ret;
 763
 764         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 &&
 765             (reg == MII_TG3_CTRL || reg == MII_TG3_AUX_CTRL))
 766                 return 0;
 767
 768         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 769                 tw32_f(MAC_MI_MODE,
 770                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 771                 udelay(80);
 772         }
 773
 774         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 775                       MI_COM_PHY_ADDR_MASK);
 776         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 777                       MI_COM_REG_ADDR_MASK);
 778         frame_val |= (val & MI_COM_DATA_MASK);
 779         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
 780
 781         tw32_f(MAC_MI_COM, frame_val);
 782
 783         loops = PHY_BUSY_LOOPS;
 784         while (loops != 0) {
 785                 udelay(10);
 786                 frame_val = tr32(MAC_MI_COM);
 787                 if ((frame_val & MI_COM_BUSY) == 0) {
 788                         udelay(5);
 789                         frame_val = tr32(MAC_MI_COM);
 790                         break;
 791                 }
 792                 loops -= 1;
 793         }
 794
 795         ret = -EBUSY;
 796         if (loops != 0)
 797                 ret = 0;
 798
 799         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 800                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 801                 udelay(80);
 802         }
 803
 804         return ret;
 805 }
 806
 807 static void tg3_phy_toggle_automdix(struct tg3 *tp, int enable)
 808 {
 809         u32 phy;
 810
 811         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
 812             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
 813                 return;
 814
 815         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
 816                 u32 ephy;
 817
 818                 if (!tg3_readphy(tp, MII_TG3_EPHY_TEST, &ephy)) {
 819                         tg3_writephy(tp, MII_TG3_EPHY_TEST,
 820                                      ephy | MII_TG3_EPHY_SHADOW_EN);
 821                         if (!tg3_readphy(tp, MII_TG3_EPHYTST_MISCCTRL, &phy)) {
 822                                 if (enable)
 823                                         phy |= MII_TG3_EPHYTST_MISCCTRL_MDIX;
 824                                 else
 825                                         phy &= ~MII_TG3_EPHYTST_MISCCTRL_MDIX;
 826                                 tg3_writephy(tp, MII_TG3_EPHYTST_MISCCTRL, phy);
 827                         }
 828                         tg3_writephy(tp, MII_TG3_EPHY_TEST, ephy);
 829                 }
 830         } else {
 831                 phy = MII_TG3_AUXCTL_MISC_RDSEL_MISC |
 832                       MII_TG3_AUXCTL_SHDWSEL_MISC;
 833                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, phy) &&
 834                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy)) {
 835                         if (enable)
 836                                 phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
 837                         else
 838                                 phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
 839                         phy |= MII_TG3_AUXCTL_MISC_WREN;
 840                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
 841                 }
 842         }
 843 }
 844
 845 static void tg3_phy_set_wirespeed(struct tg3 *tp)
 846 {
 847         u32 val;
 848
 849         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
 850                 return;
 851
 852         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
 853             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
 854                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
 855                              (val | (1 << 15) | (1 << 4)));
 856 }
 857
 858 static int tg3_bmcr_reset(struct tg3 *tp)
 859 {
 860         u32 phy_control;
 861         int limit, err;
 862
 863         /* OK, reset it, and poll the BMCR_RESET bit until it
 864          * clears or we time out.
 865          */
 866         phy_control = BMCR_RESET;
 867         err = tg3_writephy(tp, MII_BMCR, phy_control);
 868         if (err != 0)
 869                 return -EBUSY;
 870
 871         limit = 5000;
 872         while (limit--) {
 873                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
 874                 if (err != 0)
 875                         return -EBUSY;
 876
 877                 if ((phy_control & BMCR_RESET) == 0) {
 878                         udelay(40);
 879                         break;
 880                 }
 881                 udelay(10);
 882         }
 883         if (limit <= 0)
 884                 return -EBUSY;
 885
 886         return 0;
 887 }
 888
 889 static int tg3_wait_macro_done(struct tg3 *tp)
 890 {
 891         int limit = 100;
 892
 893         while (limit--) {
 894                 u32 tmp32;
 895
 896                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
 897                         if ((tmp32 & 0x1000) == 0)
 898                                 break;
 899                 }
 900         }
 901         if (limit <= 0)
 902                 return -EBUSY;
 903
 904         return 0;
 905 }
 906
 907 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
 908 {
 909         static const u32 test_pat[4][6] = {
 910         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
 911         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
 912         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
 913         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
 914         };
 915         int chan;
 916
 917         for (chan = 0; chan < 4; chan++) {
 918                 int i;
 919
 920                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 921                              (chan * 0x2000) | 0x0200);
 922                 tg3_writephy(tp, 0x16, 0x0002);
 923
 924                 for (i = 0; i < 6; i++)
 925                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
 926                                      test_pat[chan][i]);
 927
 928                 tg3_writephy(tp, 0x16, 0x0202);
 929                 if (tg3_wait_macro_done(tp)) {
 930                         *resetp = 1;
 931                         return -EBUSY;
 932                 }
 933
 934                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 935                              (chan * 0x2000) | 0x0200);
 936                 tg3_writephy(tp, 0x16, 0x0082);
 937                 if (tg3_wait_macro_done(tp)) {
 938                         *resetp = 1;
 939                         return -EBUSY;
 940                 }
 941
 942                 tg3_writephy(tp, 0x16, 0x0802);
 943                 if (tg3_wait_macro_done(tp)) {
 944                         *resetp = 1;
 945                         return -EBUSY;
 946                 }
 947
 948                 for (i = 0; i < 6; i += 2) {
 949                         u32 low, high;
 950
 951                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
 952                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
 953                             tg3_wait_macro_done(tp)) {
 954                                 *resetp = 1;
 955                                 return -EBUSY;
 956                         }
 957                         low &= 0x7fff;
 958                         high &= 0x000f;
 959                         if (low != test_pat[chan][i] ||
 960                             high != test_pat[chan][i+1]) {
 961                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
 962                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
 963                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
 964
 965                                 return -EBUSY;
 966                         }
 967                 }
 968         }
 969
 970         return 0;
 971 }
 972
 973 static int tg3_phy_reset_chanpat(struct tg3 *tp)
 974 {
 975         int chan;
 976
 977         for (chan = 0; chan < 4; chan++) {
 978                 int i;
 979
 980                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 981                              (chan * 0x2000) | 0x0200);
 982                 tg3_writephy(tp, 0x16, 0x0002);
 983                 for (i = 0; i < 6; i++)
 984                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
 985                 tg3_writephy(tp, 0x16, 0x0202);
 986                 if (tg3_wait_macro_done(tp))
 987                         return -EBUSY;
 988         }
 989
 990         return 0;
 991 }
 992
 993 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
 994 {
 995         u32 reg32, phy9_orig;
 996         int retries, do_phy_reset, err;
 997
 998         retries = 10;
 999         do_phy_reset = 1;
1000         do {
1001                 if (do_phy_reset) {
1002                         err = tg3_bmcr_reset(tp);
1003                         if (err)
1004                                 return err;
1005                         do_phy_reset = 0;
1006                 }
1007
1008                 /* Disable transmitter and interrupt.  */
1009                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
1010                         continue;
1011
1012                 reg32 |= 0x3000;
1013                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1014
1015                 /* Set full-duplex, 1000 mbps.  */
1016                 tg3_writephy(tp, MII_BMCR,
1017                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
1018
1019                 /* Set to master mode.  */
1020                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
1021                         continue;
1022
1023                 tg3_writephy(tp, MII_TG3_CTRL,
1024                              (MII_TG3_CTRL_AS_MASTER |
1025                               MII_TG3_CTRL_ENABLE_AS_MASTER));
1026
1027                 /* Enable SM_DSP_CLOCK and 6dB.  */
1028                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1029
1030                 /* Block the PHY control access.  */
1031                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1032                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
1033
1034                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
1035                 if (!err)
1036                         break;
1037         } while (--retries);
1038
1039         err = tg3_phy_reset_chanpat(tp);
1040         if (err)
1041                 return err;
1042
1043         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1044         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
1045
1046         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
1047         tg3_writephy(tp, 0x16, 0x0000);
1048
1049         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1050             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1051                 /* Set Extended packet length bit for jumbo frames */
1052                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
1053         }
1054         else {
1055                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1056         }
1057
1058         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
1059
1060         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
1061                 reg32 &= ~0x3000;
1062                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1063         } else if (!err)
1064                 err = -EBUSY;
1065
1066         return err;
1067 }
1068
1069 static void tg3_link_report(struct tg3 *);
1070
1071 /* This will reset the tigon3 PHY if there is no valid
1072  * link unless the FORCE argument is non-zero.
1073  */
1074 static int tg3_phy_reset(struct tg3 *tp)
1075 {
1076         u32 phy_status;
1077         int err;
1078
1079         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1080                 u32 val;
1081
1082                 val = tr32(GRC_MISC_CFG);
1083                 tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
1084                 udelay(40);
1085         }
1086         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
1087         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
1088         if (err != 0)
1089                 return -EBUSY;
1090
1091         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
1092                 netif_carrier_off(tp->dev);
1093                 tg3_link_report(tp);
1094         }
1095
1096         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1097             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1098             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1099                 err = tg3_phy_reset_5703_4_5(tp);
1100                 if (err)
1101                         return err;
1102                 goto out;
1103         }
1104
1105         err = tg3_bmcr_reset(tp);
1106         if (err)
1107                 return err;
1108
1109         if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
1110             tp->pci_chip_rev_id == CHIPREV_ID_5761_A0) {
1111                 u32 val;
1112
1113                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1114                 if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
1115                     CPMU_LSPD_1000MB_MACCLK_12_5) {
1116                         val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1117                         udelay(40);
1118                         tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1119                 }
1120         }
1121
1122 out:
1123         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1124                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1125                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1126                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1127                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1128                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1129                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1130         }
1131         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1132                 tg3_writephy(tp, 0x1c, 0x8d68);
1133                 tg3_writephy(tp, 0x1c, 0x8d68);
1134         }
1135         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1136                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1137                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1138                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1139                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1140                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1141                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1142                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1143                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1144         }
1145         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1146                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1147                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1148                 if (tp->tg3_flags2 & TG3_FLG2_PHY_ADJUST_TRIM) {
1149                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
1150                         tg3_writephy(tp, MII_TG3_TEST1,
1151                                      MII_TG3_TEST1_TRIM_EN | 0x4);
1152                 } else
1153                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1154                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1155         }
1156         /* Set Extended packet length bit (bit 14) on all chips that */
1157         /* support jumbo frames */
1158         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1159                 /* Cannot do read-modify-write on 5401 */
1160                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1161         } else if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1162                 u32 phy_reg;
1163
1164                 /* Set bit 14 with read-modify-write to preserve other bits */
1165                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1166                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1167                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1168         }
1169
1170         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1171          * jumbo frames transmission.
1172          */
1173         if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1174                 u32 phy_reg;
1175
1176                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1177                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1178                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1179         }
1180
1181         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1182                 /* adjust output voltage */
1183                 tg3_writephy(tp, MII_TG3_EPHY_PTEST, 0x12);
1184         }
1185
1186         tg3_phy_toggle_automdix(tp, 1);
1187         tg3_phy_set_wirespeed(tp);
1188         return 0;
1189 }
1190
1191 static void tg3_frob_aux_power(struct tg3 *tp)
1192 {
1193         struct tg3 *tp_peer = tp;
1194
1195         if ((tp->tg3_flags2 & TG3_FLG2_IS_NIC) == 0)
1196                 return;
1197
1198         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
1199             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
1200                 struct net_device *dev_peer;
1201
1202                 dev_peer = pci_get_drvdata(tp->pdev_peer);
1203                 /* remove_one() may have been run on the peer. */
1204                 if (!dev_peer)
1205                         tp_peer = tp;
1206                 else
1207                         tp_peer = netdev_priv(dev_peer);
1208         }
1209
1210         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1211             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
1212             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1213             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
1214                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1215                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1216                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1217                                     (GRC_LCLCTRL_GPIO_OE0 |
1218                                      GRC_LCLCTRL_GPIO_OE1 |
1219                                      GRC_LCLCTRL_GPIO_OE2 |
1220                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
1221                                      GRC_LCLCTRL_GPIO_OUTPUT1),
1222                                     100);
1223                 } else {
1224                         u32 no_gpio2;
1225                         u32 grc_local_ctrl = 0;
1226
1227                         if (tp_peer != tp &&
1228                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1229                                 return;
1230
1231                         /* Workaround to prevent overdrawing Amps. */
1232                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1233                             ASIC_REV_5714) {
1234                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
1235                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1236                                             grc_local_ctrl, 100);
1237                         }
1238
1239                         /* On 5753 and variants, GPIO2 cannot be used. */
1240                         no_gpio2 = tp->nic_sram_data_cfg &
1241                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
1242
1243                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
1244                                          GRC_LCLCTRL_GPIO_OE1 |
1245                                          GRC_LCLCTRL_GPIO_OE2 |
1246                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
1247                                          GRC_LCLCTRL_GPIO_OUTPUT2;
1248                         if (no_gpio2) {
1249                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
1250                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
1251                         }
1252                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1253                                                     grc_local_ctrl, 100);
1254
1255                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
1256
1257                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1258                                                     grc_local_ctrl, 100);
1259
1260                         if (!no_gpio2) {
1261                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
1262                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1263                                             grc_local_ctrl, 100);
1264                         }
1265                 }
1266         } else {
1267                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
1268                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
1269                         if (tp_peer != tp &&
1270                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1271                                 return;
1272
1273                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1274                                     (GRC_LCLCTRL_GPIO_OE1 |
1275                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1276
1277                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1278                                     GRC_LCLCTRL_GPIO_OE1, 100);
1279
1280                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1281                                     (GRC_LCLCTRL_GPIO_OE1 |
1282                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1283                 }
1284         }
1285 }
1286
1287 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
1288 {
1289         if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
1290                 return 1;
1291         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411) {
1292                 if (speed != SPEED_10)
1293                         return 1;
1294         } else if (speed == SPEED_10)
1295                 return 1;
1296
1297         return 0;
1298 }
1299
1300 static int tg3_setup_phy(struct tg3 *, int);
1301
1302 #define RESET_KIND_SHUTDOWN     0
1303 #define RESET_KIND_INIT         1
1304 #define RESET_KIND_SUSPEND      2
1305
1306 static void tg3_write_sig_post_reset(struct tg3 *, int);
1307 static int tg3_halt_cpu(struct tg3 *, u32);
1308 static int tg3_nvram_lock(struct tg3 *);
1309 static void tg3_nvram_unlock(struct tg3 *);
1310
1311 static void tg3_power_down_phy(struct tg3 *tp)
1312 {
1313         u32 val;
1314
1315         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
1316                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1317                         u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
1318                         u32 serdes_cfg = tr32(MAC_SERDES_CFG);
1319
1320                         sg_dig_ctrl |=
1321                                 SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
1322                         tw32(SG_DIG_CTRL, sg_dig_ctrl);
1323                         tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
1324                 }
1325                 return;
1326         }
1327
1328         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1329                 tg3_bmcr_reset(tp);
1330                 val = tr32(GRC_MISC_CFG);
1331                 tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
1332                 udelay(40);
1333                 return;
1334         } else {
1335                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
1336                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
1337                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x01b2);
1338         }
1339
1340         /* The PHY should not be powered down on some chips because
1341          * of bugs.
1342          */
1343         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1344             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1345             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
1346              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
1347                 return;
1348
1349         if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
1350             tp->pci_chip_rev_id == CHIPREV_ID_5761_A0) {
1351                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1352                 val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1353                 val |= CPMU_LSPD_1000MB_MACCLK_12_5;
1354                 tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1355         }
1356
1357         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
1358 }
1359
1360 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
1361 {
1362         u32 misc_host_ctrl;
1363         u16 power_control, power_caps;
1364         int pm = tp->pm_cap;
1365
1366         /* Make sure register accesses (indirect or otherwise)
1367          * will function correctly.
1368          */
1369         pci_write_config_dword(tp->pdev,
1370                                TG3PCI_MISC_HOST_CTRL,
1371                                tp->misc_host_ctrl);
1372
1373         pci_read_config_word(tp->pdev,
1374                              pm + PCI_PM_CTRL,
1375                              &power_control);
1376         power_control |= PCI_PM_CTRL_PME_STATUS;
1377         power_control &= ~(PCI_PM_CTRL_STATE_MASK);
1378         switch (state) {
1379         case PCI_D0:
1380                 power_control |= 0;
1381                 pci_write_config_word(tp->pdev,
1382                                       pm + PCI_PM_CTRL,
1383                                       power_control);
1384                 udelay(100);    /* Delay after power state change */
1385
1386                 /* Switch out of Vaux if it is a NIC */
1387                 if (tp->tg3_flags2 & TG3_FLG2_IS_NIC)
1388                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
1389
1390                 return 0;
1391
1392         case PCI_D1:
1393                 power_control |= 1;
1394                 break;
1395
1396         case PCI_D2:
1397                 power_control |= 2;
1398                 break;
1399
1400         case PCI_D3hot:
1401                 power_control |= 3;
1402                 break;
1403
1404         default:
1405                 printk(KERN_WARNING PFX "%s: Invalid power state (%d) "
1406                        "requested.\n",
1407                        tp->dev->name, state);
1408                 return -EINVAL;
1409         };
1410
1411         power_control |= PCI_PM_CTRL_PME_ENABLE;
1412
1413         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
1414         tw32(TG3PCI_MISC_HOST_CTRL,
1415              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
1416
1417         if (tp->link_config.phy_is_low_power == 0) {
1418                 tp->link_config.phy_is_low_power = 1;
1419                 tp->link_config.orig_speed = tp->link_config.speed;
1420                 tp->link_config.orig_duplex = tp->link_config.duplex;
1421                 tp->link_config.orig_autoneg = tp->link_config.autoneg;
1422         }
1423
1424         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
1425                 tp->link_config.speed = SPEED_10;
1426                 tp->link_config.duplex = DUPLEX_HALF;
1427                 tp->link_config.autoneg = AUTONEG_ENABLE;
1428                 tg3_setup_phy(tp, 0);
1429         }
1430
1431         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1432                 u32 val;
1433
1434                 val = tr32(GRC_VCPU_EXT_CTRL);
1435                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
1436         } else if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1437                 int i;
1438                 u32 val;
1439
1440                 for (i = 0; i < 200; i++) {
1441                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
1442                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1443                                 break;
1444                         msleep(1);
1445                 }
1446         }
1447         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
1448                 tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
1449                                                      WOL_DRV_STATE_SHUTDOWN |
1450                                                      WOL_DRV_WOL |
1451                                                      WOL_SET_MAGIC_PKT);
1452
1453         pci_read_config_word(tp->pdev, pm + PCI_PM_PMC, &power_caps);
1454
1455         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
1456                 u32 mac_mode;
1457
1458                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1459                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
1460                         udelay(40);
1461
1462                         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
1463                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
1464                         else
1465                                 mac_mode = MAC_MODE_PORT_MODE_MII;
1466
1467                         mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
1468                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1469                             ASIC_REV_5700) {
1470                                 u32 speed = (tp->tg3_flags &
1471                                              TG3_FLAG_WOL_SPEED_100MB) ?
1472                                              SPEED_100 : SPEED_10;
1473                                 if (tg3_5700_link_polarity(tp, speed))
1474                                         mac_mode |= MAC_MODE_LINK_POLARITY;
1475                                 else
1476                                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
1477                         }
1478                 } else {
1479                         mac_mode = MAC_MODE_PORT_MODE_TBI;
1480                 }
1481
1482                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
1483                         tw32(MAC_LED_CTRL, tp->led_ctrl);
1484
1485                 if (((power_caps & PCI_PM_CAP_PME_D3cold) &&
1486                      (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)))
1487                         mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
1488
1489                 tw32_f(MAC_MODE, mac_mode);
1490                 udelay(100);
1491
1492                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
1493                 udelay(10);
1494         }
1495
1496         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
1497             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1498              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
1499                 u32 base_val;
1500
1501                 base_val = tp->pci_clock_ctrl;
1502                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
1503                              CLOCK_CTRL_TXCLK_DISABLE);
1504
1505                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
1506                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
1507         } else if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
1508                    (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
1509                    (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)) {
1510                 /* do nothing */
1511         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
1512                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
1513                 u32 newbits1, newbits2;
1514
1515                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1516                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1517                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
1518                                     CLOCK_CTRL_TXCLK_DISABLE |
1519                                     CLOCK_CTRL_ALTCLK);
1520                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1521                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
1522                         newbits1 = CLOCK_CTRL_625_CORE;
1523                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
1524                 } else {
1525                         newbits1 = CLOCK_CTRL_ALTCLK;
1526                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1527                 }
1528
1529                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
1530                             40);
1531
1532                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
1533                             40);
1534
1535                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
1536                         u32 newbits3;
1537
1538                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1539                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1540                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
1541                                             CLOCK_CTRL_TXCLK_DISABLE |
1542                                             CLOCK_CTRL_44MHZ_CORE);
1543                         } else {
1544                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
1545                         }
1546
1547                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
1548                                     tp->pci_clock_ctrl | newbits3, 40);
1549                 }
1550         }
1551
1552         if (!(tp->tg3_flags & TG3_FLAG_WOL_ENABLE) &&
1553             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF) &&
1554             !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
1555                 tg3_power_down_phy(tp);
1556
1557         tg3_frob_aux_power(tp);
1558
1559         /* Workaround for unstable PLL clock */
1560         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
1561             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
1562                 u32 val = tr32(0x7d00);
1563
1564                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
1565                 tw32(0x7d00, val);
1566                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1567                         int err;
1568
1569                         err = tg3_nvram_lock(tp);
1570                         tg3_halt_cpu(tp, RX_CPU_BASE);
1571                         if (!err)
1572                                 tg3_nvram_unlock(tp);
1573                 }
1574         }
1575
1576         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
1577
1578         /* Finally, set the new power state. */
1579         pci_write_config_word(tp->pdev, pm + PCI_PM_CTRL, power_control);
1580         udelay(100);    /* Delay after power state change */
1581
1582         return 0;
1583 }
1584
1585 static void tg3_link_report(struct tg3 *tp)
1586 {
1587         if (!netif_carrier_ok(tp->dev)) {
1588                 if (netif_msg_link(tp))
1589                         printk(KERN_INFO PFX "%s: Link is down.\n",
1590                                tp->dev->name);
1591         } else if (netif_msg_link(tp)) {
1592                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1593                        tp->dev->name,
1594                        (tp->link_config.active_speed == SPEED_1000 ?
1595                         1000 :
1596                         (tp->link_config.active_speed == SPEED_100 ?
1597                          100 : 10)),
1598                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1599                         "full" : "half"));
1600
1601                 printk(KERN_INFO PFX "%s: Flow control is %s for TX and "
1602                        "%s for RX.\n",
1603                        tp->dev->name,
1604                        (tp->tg3_flags & TG3_FLAG_TX_PAUSE) ? "on" : "off",
1605                        (tp->tg3_flags & TG3_FLAG_RX_PAUSE) ? "on" : "off");
1606         }
1607 }
1608
1609 static void tg3_setup_flow_control(struct tg3 *tp, u32 local_adv, u32 remote_adv)
1610 {
1611         u32 new_tg3_flags = 0;
1612         u32 old_rx_mode = tp->rx_mode;
1613         u32 old_tx_mode = tp->tx_mode;
1614
1615         if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) {
1616
1617                 /* Convert 1000BaseX flow control bits to 1000BaseT
1618                  * bits before resolving flow control.
1619                  */
1620                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
1621                         local_adv &= ~(ADVERTISE_PAUSE_CAP |
1622                                        ADVERTISE_PAUSE_ASYM);
1623                         remote_adv &= ~(LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1624
1625                         if (local_adv & ADVERTISE_1000XPAUSE)
1626                                 local_adv |= ADVERTISE_PAUSE_CAP;
1627                         if (local_adv & ADVERTISE_1000XPSE_ASYM)
1628                                 local_adv |= ADVERTISE_PAUSE_ASYM;
1629                         if (remote_adv & LPA_1000XPAUSE)
1630                                 remote_adv |= LPA_PAUSE_CAP;
1631                         if (remote_adv & LPA_1000XPAUSE_ASYM)
1632                                 remote_adv |= LPA_PAUSE_ASYM;
1633                 }
1634
1635                 if (local_adv & ADVERTISE_PAUSE_CAP) {
1636                         if (local_adv & ADVERTISE_PAUSE_ASYM) {
1637                                 if (remote_adv & LPA_PAUSE_CAP)
1638                                         new_tg3_flags |=
1639                                                 (TG3_FLAG_RX_PAUSE |
1640                                                 TG3_FLAG_TX_PAUSE);
1641                                 else if (remote_adv & LPA_PAUSE_ASYM)
1642                                         new_tg3_flags |=
1643                                                 (TG3_FLAG_RX_PAUSE);
1644                         } else {
1645                                 if (remote_adv & LPA_PAUSE_CAP)
1646                                         new_tg3_flags |=
1647                                                 (TG3_FLAG_RX_PAUSE |
1648                                                 TG3_FLAG_TX_PAUSE);
1649                         }
1650                 } else if (local_adv & ADVERTISE_PAUSE_ASYM) {
1651                         if ((remote_adv & LPA_PAUSE_CAP) &&
1652                         (remote_adv & LPA_PAUSE_ASYM))
1653                                 new_tg3_flags |= TG3_FLAG_TX_PAUSE;
1654                 }
1655
1656                 tp->tg3_flags &= ~(TG3_FLAG_RX_PAUSE | TG3_FLAG_TX_PAUSE);
1657                 tp->tg3_flags |= new_tg3_flags;
1658         } else {
1659                 new_tg3_flags = tp->tg3_flags;
1660         }
1661
1662         if (new_tg3_flags & TG3_FLAG_RX_PAUSE)
1663                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1664         else
1665                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1666
1667         if (old_rx_mode != tp->rx_mode) {
1668                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1669         }
1670
1671         if (new_tg3_flags & TG3_FLAG_TX_PAUSE)
1672                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1673         else
1674                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1675
1676         if (old_tx_mode != tp->tx_mode) {
1677                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1678         }
1679 }
1680
1681 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
1682 {
1683         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
1684         case MII_TG3_AUX_STAT_10HALF:
1685                 *speed = SPEED_10;
1686                 *duplex = DUPLEX_HALF;
1687                 break;
1688
1689         case MII_TG3_AUX_STAT_10FULL:
1690                 *speed = SPEED_10;
1691                 *duplex = DUPLEX_FULL;
1692                 break;
1693
1694         case MII_TG3_AUX_STAT_100HALF:
1695                 *speed = SPEED_100;
1696                 *duplex = DUPLEX_HALF;
1697                 break;
1698
1699         case MII_TG3_AUX_STAT_100FULL:
1700                 *speed = SPEED_100;
1701                 *duplex = DUPLEX_FULL;
1702                 break;
1703
1704         case MII_TG3_AUX_STAT_1000HALF:
1705                 *speed = SPEED_1000;
1706                 *duplex = DUPLEX_HALF;
1707                 break;
1708
1709         case MII_TG3_AUX_STAT_1000FULL:
1710                 *speed = SPEED_1000;
1711                 *duplex = DUPLEX_FULL;
1712                 break;
1713
1714         default:
1715                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1716                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
1717                                  SPEED_10;
1718                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
1719                                   DUPLEX_HALF;
1720                         break;
1721                 }
1722                 *speed = SPEED_INVALID;
1723                 *duplex = DUPLEX_INVALID;
1724                 break;
1725         };
1726 }
1727
1728 static void tg3_phy_copper_begin(struct tg3 *tp)
1729 {
1730         u32 new_adv;
1731         int i;
1732
1733         if (tp->link_config.phy_is_low_power) {
1734                 /* Entering low power mode.  Disable gigabit and
1735                  * 100baseT advertisements.
1736                  */
1737                 tg3_writephy(tp, MII_TG3_CTRL, 0);
1738
1739                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1740                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1741                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
1742                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
1743
1744                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1745         } else if (tp->link_config.speed == SPEED_INVALID) {
1746                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
1747                         tp->link_config.advertising &=
1748                                 ~(ADVERTISED_1000baseT_Half |
1749                                   ADVERTISED_1000baseT_Full);
1750
1751                 new_adv = (ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1752                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
1753                         new_adv |= ADVERTISE_10HALF;
1754                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
1755                         new_adv |= ADVERTISE_10FULL;
1756                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
1757                         new_adv |= ADVERTISE_100HALF;
1758                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
1759                         new_adv |= ADVERTISE_100FULL;
1760                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1761
1762                 if (tp->link_config.advertising &
1763                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
1764                         new_adv = 0;
1765                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
1766                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
1767                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
1768                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
1769                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
1770                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1771                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
1772                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1773                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1774                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1775                 } else {
1776                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1777                 }
1778         } else {
1779                 /* Asking for a specific link mode. */
1780                 if (tp->link_config.speed == SPEED_1000) {
1781                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1782                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1783
1784                         if (tp->link_config.duplex == DUPLEX_FULL)
1785                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
1786                         else
1787                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
1788                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1789                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
1790                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1791                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1792                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1793                 } else {
1794                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1795
1796                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1797                         if (tp->link_config.speed == SPEED_100) {
1798                                 if (tp->link_config.duplex == DUPLEX_FULL)
1799                                         new_adv |= ADVERTISE_100FULL;
1800                                 else
1801                                         new_adv |= ADVERTISE_100HALF;
1802                         } else {
1803                                 if (tp->link_config.duplex == DUPLEX_FULL)
1804                                         new_adv |= ADVERTISE_10FULL;
1805                                 else
1806                                         new_adv |= ADVERTISE_10HALF;
1807                         }
1808                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1809                 }
1810         }
1811
1812         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
1813             tp->link_config.speed != SPEED_INVALID) {
1814                 u32 bmcr, orig_bmcr;
1815
1816                 tp->link_config.active_speed = tp->link_config.speed;
1817                 tp->link_config.active_duplex = tp->link_config.duplex;
1818
1819                 bmcr = 0;
1820                 switch (tp->link_config.speed) {
1821                 default:
1822                 case SPEED_10:
1823                         break;
1824
1825                 case SPEED_100:
1826                         bmcr |= BMCR_SPEED100;
1827                         break;
1828
1829                 case SPEED_1000:
1830                         bmcr |= TG3_BMCR_SPEED1000;
1831                         break;
1832                 };
1833
1834                 if (tp->link_config.duplex == DUPLEX_FULL)
1835                         bmcr |= BMCR_FULLDPLX;
1836
1837                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
1838                     (bmcr != orig_bmcr)) {
1839                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
1840                         for (i = 0; i < 1500; i++) {
1841                                 u32 tmp;
1842
1843                                 udelay(10);
1844                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
1845                                     tg3_readphy(tp, MII_BMSR, &tmp))
1846                                         continue;
1847                                 if (!(tmp & BMSR_LSTATUS)) {
1848                                         udelay(40);
1849                                         break;
1850                                 }
1851                         }
1852                         tg3_writephy(tp, MII_BMCR, bmcr);
1853                         udelay(40);
1854                 }
1855         } else {
1856                 tg3_writephy(tp, MII_BMCR,
1857                              BMCR_ANENABLE | BMCR_ANRESTART);
1858         }
1859 }
1860
1861 static int tg3_init_5401phy_dsp(struct tg3 *tp)
1862 {
1863         int err;
1864
1865         /* Turn off tap power management. */
1866         /* Set Extended packet length bit */
1867         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1868
1869         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
1870         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
1871
1872         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
1873         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
1874
1875         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1876         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
1877
1878         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1879         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
1880
1881         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1882         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
1883
1884         udelay(40);
1885
1886         return err;
1887 }
1888
1889 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
1890 {
1891         u32 adv_reg, all_mask = 0;
1892
1893         if (mask & ADVERTISED_10baseT_Half)
1894                 all_mask |= ADVERTISE_10HALF;
1895         if (mask & ADVERTISED_10baseT_Full)
1896                 all_mask |= ADVERTISE_10FULL;
1897         if (mask & ADVERTISED_100baseT_Half)
1898                 all_mask |= ADVERTISE_100HALF;
1899         if (mask & ADVERTISED_100baseT_Full)
1900                 all_mask |= ADVERTISE_100FULL;
1901
1902         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
1903                 return 0;
1904
1905         if ((adv_reg & all_mask) != all_mask)
1906                 return 0;
1907         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1908                 u32 tg3_ctrl;
1909
1910                 all_mask = 0;
1911                 if (mask & ADVERTISED_1000baseT_Half)
1912                         all_mask |= ADVERTISE_1000HALF;
1913                 if (mask & ADVERTISED_1000baseT_Full)
1914                         all_mask |= ADVERTISE_1000FULL;
1915
1916                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
1917                         return 0;
1918
1919                 if ((tg3_ctrl & all_mask) != all_mask)
1920                         return 0;
1921         }
1922         return 1;
1923 }
1924
1925 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
1926 {
1927         int current_link_up;
1928         u32 bmsr, dummy;
1929         u16 current_speed;
1930         u8 current_duplex;
1931         int i, err;
1932
1933         tw32(MAC_EVENT, 0);
1934
1935         tw32_f(MAC_STATUS,
1936              (MAC_STATUS_SYNC_CHANGED |
1937               MAC_STATUS_CFG_CHANGED |
1938               MAC_STATUS_MI_COMPLETION |
1939               MAC_STATUS_LNKSTATE_CHANGED));
1940         udelay(40);
1941
1942         tp->mi_mode = MAC_MI_MODE_BASE;
1943         tw32_f(MAC_MI_MODE, tp->mi_mode);
1944         udelay(80);
1945
1946         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
1947
1948         /* Some third-party PHYs need to be reset on link going
1949          * down.
1950          */
1951         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1952              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1953              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
1954             netif_carrier_ok(tp->dev)) {
1955                 tg3_readphy(tp, MII_BMSR, &bmsr);
1956                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1957                     !(bmsr & BMSR_LSTATUS))
1958                         force_reset = 1;
1959         }
1960         if (force_reset)
1961                 tg3_phy_reset(tp);
1962
1963         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1964                 tg3_readphy(tp, MII_BMSR, &bmsr);
1965                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
1966                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
1967                         bmsr = 0;
1968
1969                 if (!(bmsr & BMSR_LSTATUS)) {
1970                         err = tg3_init_5401phy_dsp(tp);
1971                         if (err)
1972                                 return err;
1973
1974                         tg3_readphy(tp, MII_BMSR, &bmsr);
1975                         for (i = 0; i < 1000; i++) {
1976                                 udelay(10);
1977                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1978                                     (bmsr & BMSR_LSTATUS)) {
1979                                         udelay(40);
1980                                         break;
1981                                 }
1982                         }
1983
1984                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
1985                             !(bmsr & BMSR_LSTATUS) &&
1986                             tp->link_config.active_speed == SPEED_1000) {
1987                                 err = tg3_phy_reset(tp);
1988                                 if (!err)
1989                                         err = tg3_init_5401phy_dsp(tp);
1990                                 if (err)
1991                                         return err;
1992                         }
1993                 }
1994         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1995                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
1996                 /* 5701 {A0,B0} CRC bug workaround */
1997                 tg3_writephy(tp, 0x15, 0x0a75);
1998                 tg3_writephy(tp, 0x1c, 0x8c68);
1999                 tg3_writephy(tp, 0x1c, 0x8d68);
2000                 tg3_writephy(tp, 0x1c, 0x8c68);
2001         }
2002
2003         /* Clear pending interrupts... */
2004         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
2005         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
2006
2007         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
2008                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
2009         else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
2010                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
2011
2012         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2013             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2014                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
2015                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
2016                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
2017                 else
2018                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
2019         }
2020
2021         current_link_up = 0;
2022         current_speed = SPEED_INVALID;
2023         current_duplex = DUPLEX_INVALID;
2024
2025         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
2026                 u32 val;
2027
2028                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
2029                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
2030                 if (!(val & (1 << 10))) {
2031                         val |= (1 << 10);
2032                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
2033                         goto relink;
2034                 }
2035         }
2036
2037         bmsr = 0;
2038         for (i = 0; i < 100; i++) {
2039                 tg3_readphy(tp, MII_BMSR, &bmsr);
2040                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2041                     (bmsr & BMSR_LSTATUS))
2042                         break;
2043                 udelay(40);
2044         }
2045
2046         if (bmsr & BMSR_LSTATUS) {
2047                 u32 aux_stat, bmcr;
2048
2049                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
2050                 for (i = 0; i < 2000; i++) {
2051                         udelay(10);
2052                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
2053                             aux_stat)
2054                                 break;
2055                 }
2056
2057                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
2058                                              &current_speed,
2059                                              &current_duplex);
2060
2061                 bmcr = 0;
2062                 for (i = 0; i < 200; i++) {
2063                         tg3_readphy(tp, MII_BMCR, &bmcr);
2064                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
2065                                 continue;
2066                         if (bmcr && bmcr != 0x7fff)
2067                                 break;
2068                         udelay(10);
2069                 }
2070
2071                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2072                         if (bmcr & BMCR_ANENABLE) {
2073                                 current_link_up = 1;
2074
2075                                 /* Force autoneg restart if we are exiting
2076                                  * low power mode.
2077                                  */
2078                                 if (!tg3_copper_is_advertising_all(tp,
2079                                                 tp->link_config.advertising))
2080                                         current_link_up = 0;
2081                         } else {
2082                                 current_link_up = 0;
2083                         }
2084                 } else {
2085                         if (!(bmcr & BMCR_ANENABLE) &&
2086                             tp->link_config.speed == current_speed &&
2087                             tp->link_config.duplex == current_duplex) {
2088                                 current_link_up = 1;
2089                         } else {
2090                                 current_link_up = 0;
2091                         }
2092                 }
2093
2094                 tp->link_config.active_speed = current_speed;
2095                 tp->link_config.active_duplex = current_duplex;
2096         }
2097
2098         if (current_link_up == 1 &&
2099             (tp->link_config.active_duplex == DUPLEX_FULL) &&
2100             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
2101                 u32 local_adv, remote_adv;
2102
2103                 if (tg3_readphy(tp, MII_ADVERTISE, &local_adv))
2104                         local_adv = 0;
2105                 local_adv &= (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
2106
2107                 if (tg3_readphy(tp, MII_LPA, &remote_adv))
2108                         remote_adv = 0;
2109
2110                 remote_adv &= (LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
2111
2112                 /* If we are not advertising full pause capability,
2113                  * something is wrong.  Bring the link down and reconfigure.
2114                  */
2115                 if (local_adv != ADVERTISE_PAUSE_CAP) {
2116                         current_link_up = 0;
2117                 } else {
2118                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2119                 }
2120         }
2121 relink:
2122         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
2123                 u32 tmp;
2124
2125                 tg3_phy_copper_begin(tp);
2126
2127                 tg3_readphy(tp, MII_BMSR, &tmp);
2128                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
2129                     (tmp & BMSR_LSTATUS))
2130                         current_link_up = 1;
2131         }
2132
2133         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
2134         if (current_link_up == 1) {
2135                 if (tp->link_config.active_speed == SPEED_100 ||
2136                     tp->link_config.active_speed == SPEED_10)
2137                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
2138                 else
2139                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2140         } else
2141                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2142
2143         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
2144         if (tp->link_config.active_duplex == DUPLEX_HALF)
2145                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
2146
2147         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
2148                 if (current_link_up == 1 &&
2149                     tg3_5700_link_polarity(tp, tp->link_config.active_speed))
2150                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
2151                 else
2152                         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
2153         }
2154
2155         /* ??? Without this setting Netgear GA302T PHY does not
2156          * ??? send/receive packets...
2157          */
2158         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
2159             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
2160                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
2161                 tw32_f(MAC_MI_MODE, tp->mi_mode);
2162                 udelay(80);
2163         }
2164
2165         tw32_f(MAC_MODE, tp->mac_mode);
2166         udelay(40);
2167
2168         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
2169                 /* Polled via timer. */
2170                 tw32_f(MAC_EVENT, 0);
2171         } else {
2172                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2173         }
2174         udelay(40);
2175
2176         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
2177             current_link_up == 1 &&
2178             tp->link_config.active_speed == SPEED_1000 &&
2179             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
2180              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
2181                 udelay(120);
2182                 tw32_f(MAC_STATUS,
2183                      (MAC_STATUS_SYNC_CHANGED |
2184                       MAC_STATUS_CFG_CHANGED));
2185                 udelay(40);
2186                 tg3_write_mem(tp,
2187                               NIC_SRAM_FIRMWARE_MBOX,
2188                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
2189         }
2190
2191         if (current_link_up != netif_carrier_ok(tp->dev)) {
2192                 if (current_link_up)
2193                         netif_carrier_on(tp->dev);
2194                 else
2195                         netif_carrier_off(tp->dev);
2196                 tg3_link_report(tp);
2197         }
2198
2199         return 0;
2200 }
2201
2202 struct tg3_fiber_aneginfo {
2203         int state;
2204 #define ANEG_STATE_UNKNOWN              0
2205 #define ANEG_STATE_AN_ENABLE            1
2206 #define ANEG_STATE_RESTART_INIT         2
2207 #define ANEG_STATE_RESTART              3
2208 #define ANEG_STATE_DISABLE_LINK_OK      4
2209 #define ANEG_STATE_ABILITY_DETECT_INIT  5
2210 #define ANEG_STATE_ABILITY_DETECT       6
2211 #define ANEG_STATE_ACK_DETECT_INIT      7
2212 #define ANEG_STATE_ACK_DETECT           8
2213 #define ANEG_STATE_COMPLETE_ACK_INIT    9
2214 #define ANEG_STATE_COMPLETE_ACK         10
2215 #define ANEG_STATE_IDLE_DETECT_INIT     11
2216 #define ANEG_STATE_IDLE_DETECT          12
2217 #define ANEG_STATE_LINK_OK              13
2218 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
2219 #define ANEG_STATE_NEXT_PAGE_WAIT       15
2220
2221         u32 flags;
2222 #define MR_AN_ENABLE            0x00000001
2223 #define MR_RESTART_AN           0x00000002
2224 #define MR_AN_COMPLETE          0x00000004
2225 #define MR_PAGE_RX              0x00000008
2226 #define MR_NP_LOADED            0x00000010
2227 #define MR_TOGGLE_TX            0x00000020
2228 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
2229 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
2230 #define MR_LP_ADV_SYM_PAUSE     0x00000100
2231 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
2232 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
2233 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
2234 #define MR_LP_ADV_NEXT_PAGE     0x00001000
2235 #define MR_TOGGLE_RX            0x00002000
2236 #define MR_NP_RX                0x00004000
2237
2238 #define MR_LINK_OK              0x80000000
2239
2240         unsigned long link_time, cur_time;
2241
2242         u32 ability_match_cfg;
2243         int ability_match_count;
2244
2245         char ability_match, idle_match, ack_match;
2246
2247         u32 txconfig, rxconfig;
2248 #define ANEG_CFG_NP             0x00000080
2249 #define ANEG_CFG_ACK            0x00000040
2250 #define ANEG_CFG_RF2            0x00000020
2251 #define ANEG_CFG_RF1            0x00000010
2252 #define ANEG_CFG_PS2            0x00000001
2253 #define ANEG_CFG_PS1            0x00008000
2254 #define ANEG_CFG_HD             0x00004000
2255 #define ANEG_CFG_FD             0x00002000
2256 #define ANEG_CFG_INVAL          0x00001f06
2257
2258 };
2259 #define ANEG_OK         0
2260 #define ANEG_DONE       1
2261 #define ANEG_TIMER_ENAB 2
2262 #define ANEG_FAILED     -1
2263
2264 #define ANEG_STATE_SETTLE_TIME  10000
2265
2266 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
2267                                    struct tg3_fiber_aneginfo *ap)
2268 {
2269         unsigned long delta;
2270         u32 rx_cfg_reg;
2271         int ret;
2272
2273         if (ap->state == ANEG_STATE_UNKNOWN) {
2274                 ap->rxconfig = 0;
2275                 ap->link_time = 0;
2276                 ap->cur_time = 0;
2277                 ap->ability_match_cfg = 0;
2278                 ap->ability_match_count = 0;
2279                 ap->ability_match = 0;
2280                 ap->idle_match = 0;
2281                 ap->ack_match = 0;
2282         }
2283         ap->cur_time++;
2284
2285         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
2286                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
2287
2288                 if (rx_cfg_reg != ap->ability_match_cfg) {
2289                         ap->ability_match_cfg = rx_cfg_reg;
2290                         ap->ability_match = 0;
2291                         ap->ability_match_count = 0;
2292                 } else {
2293                         if (++ap->ability_match_count > 1) {
2294                                 ap->ability_match = 1;
2295                                 ap->ability_match_cfg = rx_cfg_reg;
2296                         }
2297                 }
2298                 if (rx_cfg_reg & ANEG_CFG_ACK)
2299                         ap->ack_match = 1;
2300                 else
2301                         ap->ack_match = 0;
2302
2303                 ap->idle_match = 0;
2304         } else {
2305                 ap->idle_match = 1;
2306                 ap->ability_match_cfg = 0;
2307                 ap->ability_match_count = 0;
2308                 ap->ability_match = 0;
2309                 ap->ack_match = 0;
2310
2311                 rx_cfg_reg = 0;
2312         }
2313
2314         ap->rxconfig = rx_cfg_reg;
2315         ret = ANEG_OK;
2316
2317         switch(ap->state) {
2318         case ANEG_STATE_UNKNOWN:
2319                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
2320                         ap->state = ANEG_STATE_AN_ENABLE;
2321
2322                 /* fallthru */
2323         case ANEG_STATE_AN_ENABLE:
2324                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
2325                 if (ap->flags & MR_AN_ENABLE) {
2326                         ap->link_time = 0;
2327                         ap->cur_time = 0;
2328                         ap->ability_match_cfg = 0;
2329                         ap->ability_match_count = 0;
2330                         ap->ability_match = 0;
2331                         ap->idle_match = 0;
2332                         ap->ack_match = 0;
2333
2334                         ap->state = ANEG_STATE_RESTART_INIT;
2335                 } else {
2336                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
2337                 }
2338                 break;
2339
2340         case ANEG_STATE_RESTART_INIT:
2341                 ap->link_time = ap->cur_time;
2342                 ap->flags &= ~(MR_NP_LOADED);
2343                 ap->txconfig = 0;
2344                 tw32(MAC_TX_AUTO_NEG, 0);
2345                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2346                 tw32_f(MAC_MODE, tp->mac_mode);
2347                 udelay(40);
2348
2349                 ret = ANEG_TIMER_ENAB;
2350                 ap->state = ANEG_STATE_RESTART;
2351
2352                 /* fallthru */
2353         case ANEG_STATE_RESTART:
2354                 delta = ap->cur_time - ap->link_time;
2355                 if (delta > ANEG_STATE_SETTLE_TIME) {
2356                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
2357                 } else {
2358                         ret = ANEG_TIMER_ENAB;
2359                 }
2360                 break;
2361
2362         case ANEG_STATE_DISABLE_LINK_OK:
2363                 ret = ANEG_DONE;
2364                 break;
2365
2366         case ANEG_STATE_ABILITY_DETECT_INIT:
2367                 ap->flags &= ~(MR_TOGGLE_TX);
2368                 ap->txconfig = (ANEG_CFG_FD | ANEG_CFG_PS1);
2369                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2370                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2371                 tw32_f(MAC_MODE, tp->mac_mode);
2372                 udelay(40);
2373
2374                 ap->state = ANEG_STATE_ABILITY_DETECT;
2375                 break;
2376
2377         case ANEG_STATE_ABILITY_DETECT:
2378                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
2379                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
2380                 }
2381                 break;
2382
2383         case ANEG_STATE_ACK_DETECT_INIT:
2384                 ap->txconfig |= ANEG_CFG_ACK;
2385                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2386                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2387                 tw32_f(MAC_MODE, tp->mac_mode);
2388                 udelay(40);
2389
2390                 ap->state = ANEG_STATE_ACK_DETECT;
2391
2392                 /* fallthru */
2393         case ANEG_STATE_ACK_DETECT:
2394                 if (ap->ack_match != 0) {
2395                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
2396                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
2397                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
2398                         } else {
2399                                 ap->state = ANEG_STATE_AN_ENABLE;
2400                         }
2401                 } else if (ap->ability_match != 0 &&
2402                            ap->rxconfig == 0) {
2403                         ap->state = ANEG_STATE_AN_ENABLE;
2404                 }
2405                 break;
2406
2407         case ANEG_STATE_COMPLETE_ACK_INIT:
2408                 if (ap->rxconfig & ANEG_CFG_INVAL) {
2409                         ret = ANEG_FAILED;
2410                         break;
2411                 }
2412                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
2413                                MR_LP_ADV_HALF_DUPLEX |
2414                                MR_LP_ADV_SYM_PAUSE |
2415                                MR_LP_ADV_ASYM_PAUSE |
2416                                MR_LP_ADV_REMOTE_FAULT1 |
2417                                MR_LP_ADV_REMOTE_FAULT2 |
2418                                MR_LP_ADV_NEXT_PAGE |
2419                                MR_TOGGLE_RX |
2420                                MR_NP_RX);
2421                 if (ap->rxconfig & ANEG_CFG_FD)
2422                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
2423                 if (ap->rxconfig & ANEG_CFG_HD)
2424                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
2425                 if (ap->rxconfig & ANEG_CFG_PS1)
2426                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
2427                 if (ap->rxconfig & ANEG_CFG_PS2)
2428                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
2429                 if (ap->rxconfig & ANEG_CFG_RF1)
2430                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
2431                 if (ap->rxconfig & ANEG_CFG_RF2)
2432                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
2433                 if (ap->rxconfig & ANEG_CFG_NP)
2434                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
2435
2436                 ap->link_time = ap->cur_time;
2437
2438                 ap->flags ^= (MR_TOGGLE_TX);
2439                 if (ap->rxconfig & 0x0008)
2440                         ap->flags |= MR_TOGGLE_RX;
2441                 if (ap->rxconfig & ANEG_CFG_NP)
2442                         ap->flags |= MR_NP_RX;
2443                 ap->flags |= MR_PAGE_RX;
2444
2445                 ap->state = ANEG_STATE_COMPLETE_ACK;
2446                 ret = ANEG_TIMER_ENAB;
2447                 break;
2448
2449         case ANEG_STATE_COMPLETE_ACK:
2450                 if (ap->ability_match != 0 &&
2451                     ap->rxconfig == 0) {
2452                         ap->state = ANEG_STATE_AN_ENABLE;
2453                         break;
2454                 }
2455                 delta = ap->cur_time - ap->link_time;
2456                 if (delta > ANEG_STATE_SETTLE_TIME) {
2457                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
2458                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2459                         } else {
2460                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
2461                                     !(ap->flags & MR_NP_RX)) {
2462                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2463                                 } else {
2464                                         ret = ANEG_FAILED;
2465                                 }
2466                         }
2467                 }
2468                 break;
2469
2470         case ANEG_STATE_IDLE_DETECT_INIT:
2471                 ap->link_time = ap->cur_time;
2472                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2473                 tw32_f(MAC_MODE, tp->mac_mode);
2474                 udelay(40);
2475
2476                 ap->state = ANEG_STATE_IDLE_DETECT;
2477                 ret = ANEG_TIMER_ENAB;
2478                 break;
2479
2480         case ANEG_STATE_IDLE_DETECT:
2481                 if (ap->ability_match != 0 &&
2482                     ap->rxconfig == 0) {
2483                         ap->state = ANEG_STATE_AN_ENABLE;
2484                         break;
2485                 }
2486                 delta = ap->cur_time - ap->link_time;
2487                 if (delta > ANEG_STATE_SETTLE_TIME) {
2488                         /* XXX another gem from the Broadcom driver :( */
2489                         ap->state = ANEG_STATE_LINK_OK;
2490                 }
2491                 break;
2492
2493         case ANEG_STATE_LINK_OK:
2494                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
2495                 ret = ANEG_DONE;
2496                 break;
2497
2498         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
2499                 /* ??? unimplemented */
2500                 break;
2501
2502         case ANEG_STATE_NEXT_PAGE_WAIT:
2503                 /* ??? unimplemented */
2504                 break;
2505
2506         default:
2507                 ret = ANEG_FAILED;
2508                 break;
2509         };
2510
2511         return ret;
2512 }
2513
2514 static int fiber_autoneg(struct tg3 *tp, u32 *flags)
2515 {
2516         int res = 0;
2517         struct tg3_fiber_aneginfo aninfo;
2518         int status = ANEG_FAILED;
2519         unsigned int tick;
2520         u32 tmp;
2521
2522         tw32_f(MAC_TX_AUTO_NEG, 0);
2523
2524         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
2525         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
2526         udelay(40);
2527
2528         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
2529         udelay(40);
2530
2531         memset(&aninfo, 0, sizeof(aninfo));
2532         aninfo.flags |= MR_AN_ENABLE;
2533         aninfo.state = ANEG_STATE_UNKNOWN;
2534         aninfo.cur_time = 0;
2535         tick = 0;
2536         while (++tick < 195000) {
2537                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
2538                 if (status == ANEG_DONE || status == ANEG_FAILED)
2539                         break;
2540
2541                 udelay(1);
2542         }
2543
2544         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2545         tw32_f(MAC_MODE, tp->mac_mode);
2546         udelay(40);
2547
2548         *flags = aninfo.flags;
2549
2550         if (status == ANEG_DONE &&
2551             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
2552                              MR_LP_ADV_FULL_DUPLEX)))
2553                 res = 1;
2554
2555         return res;
2556 }
2557
2558 static void tg3_init_bcm8002(struct tg3 *tp)
2559 {
2560         u32 mac_status = tr32(MAC_STATUS);
2561         int i;
2562
2563         /* Reset when initting first time or we have a link. */
2564         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
2565             !(mac_status & MAC_STATUS_PCS_SYNCED))
2566                 return;
2567
2568         /* Set PLL lock range. */
2569         tg3_writephy(tp, 0x16, 0x8007);
2570
2571         /* SW reset */
2572         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
2573
2574         /* Wait for reset to complete. */
2575         /* XXX schedule_timeout() ... */
2576         for (i = 0; i < 500; i++)
2577                 udelay(10);
2578
2579         /* Config mode; select PMA/Ch 1 regs. */
2580         tg3_writephy(tp, 0x10, 0x8411);
2581
2582         /* Enable auto-lock and comdet, select txclk for tx. */
2583         tg3_writephy(tp, 0x11, 0x0a10);
2584
2585         tg3_writephy(tp, 0x18, 0x00a0);
2586         tg3_writephy(tp, 0x16, 0x41ff);
2587
2588         /* Assert and deassert POR. */
2589         tg3_writephy(tp, 0x13, 0x0400);
2590         udelay(40);
2591         tg3_writephy(tp, 0x13, 0x0000);
2592
2593         tg3_writephy(tp, 0x11, 0x0a50);
2594         udelay(40);
2595         tg3_writephy(tp, 0x11, 0x0a10);
2596
2597         /* Wait for signal to stabilize */
2598         /* XXX schedule_timeout() ... */
2599         for (i = 0; i < 15000; i++)
2600                 udelay(10);
2601
2602         /* Deselect the channel register so we can read the PHYID
2603          * later.
2604          */
2605         tg3_writephy(tp, 0x10, 0x8011);
2606 }
2607
2608 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
2609 {
2610         u32 sg_dig_ctrl, sg_dig_status;
2611         u32 serdes_cfg, expected_sg_dig_ctrl;
2612         int workaround, port_a;
2613         int current_link_up;
2614
2615         serdes_cfg = 0;
2616         expected_sg_dig_ctrl = 0;
2617         workaround = 0;
2618         port_a = 1;
2619         current_link_up = 0;
2620
2621         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
2622             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
2623                 workaround = 1;
2624                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
2625                         port_a = 0;
2626
2627                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
2628                 /* preserve bits 20-23 for voltage regulator */
2629                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
2630         }
2631
2632         sg_dig_ctrl = tr32(SG_DIG_CTRL);
2633
2634         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
2635                 if (sg_dig_ctrl & (1 << 31)) {
2636                         if (workaround) {
2637                                 u32 val = serdes_cfg;
2638
2639                                 if (port_a)
2640                                         val |= 0xc010000;
2641                                 else
2642                                         val |= 0x4010000;
2643                                 tw32_f(MAC_SERDES_CFG, val);
2644                         }
2645                         tw32_f(SG_DIG_CTRL, 0x01388400);
2646                 }
2647                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
2648                         tg3_setup_flow_control(tp, 0, 0);
2649                         current_link_up = 1;
2650                 }
2651                 goto out;
2652         }
2653
2654         /* Want auto-negotiation.  */
2655         expected_sg_dig_ctrl = 0x81388400;
2656
2657         /* Pause capability */
2658         expected_sg_dig_ctrl |= (1 << 11);
2659
2660         /* Asymettric pause */
2661         expected_sg_dig_ctrl |= (1 << 12);
2662
2663         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
2664                 if ((tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
2665                     tp->serdes_counter &&
2666                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
2667                                     MAC_STATUS_RCVD_CFG)) ==
2668                      MAC_STATUS_PCS_SYNCED)) {
2669                         tp->serdes_counter--;
2670                         current_link_up = 1;
2671                         goto out;
2672                 }
2673 restart_autoneg:
2674                 if (workaround)
2675                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
2676                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | (1 << 30));
2677                 udelay(5);
2678                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
2679
2680                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
2681                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2682         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
2683                                  MAC_STATUS_SIGNAL_DET)) {
2684                 sg_dig_status = tr32(SG_DIG_STATUS);
2685                 mac_status = tr32(MAC_STATUS);
2686
2687                 if ((sg_dig_status & (1 << 1)) &&
2688                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
2689                         u32 local_adv, remote_adv;
2690
2691                         local_adv = ADVERTISE_PAUSE_CAP;
2692                         remote_adv = 0;
2693                         if (sg_dig_status & (1 << 19))
2694                                 remote_adv |= LPA_PAUSE_CAP;
2695                         if (sg_dig_status & (1 << 20))
2696                                 remote_adv |= LPA_PAUSE_ASYM;
2697
2698                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2699                         current_link_up = 1;
2700                         tp->serdes_counter = 0;
2701                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2702                 } else if (!(sg_dig_status & (1 << 1))) {
2703                         if (tp->serdes_counter)
2704                                 tp->serdes_counter--;
2705                         else {
2706                                 if (workaround) {
2707                                         u32 val = serdes_cfg;
2708
2709                                         if (port_a)
2710                                                 val |= 0xc010000;
2711                                         else
2712                                                 val |= 0x4010000;
2713
2714                                         tw32_f(MAC_SERDES_CFG, val);
2715                                 }
2716
2717                                 tw32_f(SG_DIG_CTRL, 0x01388400);
2718                                 udelay(40);
2719
2720                                 /* Link parallel detection - link is up */
2721                                 /* only if we have PCS_SYNC and not */
2722                                 /* receiving config code words */
2723                                 mac_status = tr32(MAC_STATUS);
2724                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
2725                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
2726                                         tg3_setup_flow_control(tp, 0, 0);
2727                                         current_link_up = 1;
2728                                         tp->tg3_flags2 |=
2729                                                 TG3_FLG2_PARALLEL_DETECT;
2730                                         tp->serdes_counter =
2731                                                 SERDES_PARALLEL_DET_TIMEOUT;
2732                                 } else
2733                                         goto restart_autoneg;
2734                         }
2735                 }
2736         } else {
2737                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
2738                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2739         }
2740
2741 out:
2742         return current_link_up;
2743 }
2744
2745 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
2746 {
2747         int current_link_up = 0;
2748
2749         if (!(mac_status & MAC_STATUS_PCS_SYNCED))
2750                 goto out;
2751
2752         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2753                 u32 flags;
2754                 int i;
2755
2756                 if (fiber_autoneg(tp, &flags)) {
2757                         u32 local_adv, remote_adv;
2758
2759                         local_adv = ADVERTISE_PAUSE_CAP;
2760                         remote_adv = 0;
2761                         if (flags & MR_LP_ADV_SYM_PAUSE)
2762                                 remote_adv |= LPA_PAUSE_CAP;
2763                         if (flags & MR_LP_ADV_ASYM_PAUSE)
2764                                 remote_adv |= LPA_PAUSE_ASYM;
2765
2766                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2767
2768                         current_link_up = 1;
2769                 }
2770                 for (i = 0; i < 30; i++) {
2771                         udelay(20);
2772                         tw32_f(MAC_STATUS,
2773                                (MAC_STATUS_SYNC_CHANGED |
2774                                 MAC_STATUS_CFG_CHANGED));
2775                         udelay(40);
2776                         if ((tr32(MAC_STATUS) &
2777                              (MAC_STATUS_SYNC_CHANGED |
2778                               MAC_STATUS_CFG_CHANGED)) == 0)
2779                                 break;
2780                 }
2781
2782                 mac_status = tr32(MAC_STATUS);
2783                 if (current_link_up == 0 &&
2784                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
2785                     !(mac_status & MAC_STATUS_RCVD_CFG))
2786                         current_link_up = 1;
2787         } else {
2788                 /* Forcing 1000FD link up. */
2789                 current_link_up = 1;
2790
2791                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
2792                 udelay(40);
2793
2794                 tw32_f(MAC_MODE, tp->mac_mode);
2795                 udelay(40);
2796         }
2797
2798 out:
2799         return current_link_up;
2800 }
2801
2802 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
2803 {
2804         u32 orig_pause_cfg;
2805         u16 orig_active_speed;
2806         u8 orig_active_duplex;
2807         u32 mac_status;
2808         int current_link_up;
2809         int i;
2810
2811         orig_pause_cfg =
2812                 (tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2813                                   TG3_FLAG_TX_PAUSE));
2814         orig_active_speed = tp->link_config.active_speed;
2815         orig_active_duplex = tp->link_config.active_duplex;
2816
2817         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
2818             netif_carrier_ok(tp->dev) &&
2819             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
2820                 mac_status = tr32(MAC_STATUS);
2821                 mac_status &= (MAC_STATUS_PCS_SYNCED |
2822                                MAC_STATUS_SIGNAL_DET |
2823                                MAC_STATUS_CFG_CHANGED |
2824                                MAC_STATUS_RCVD_CFG);
2825                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
2826                                    MAC_STATUS_SIGNAL_DET)) {
2827                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2828                                             MAC_STATUS_CFG_CHANGED));
2829                         return 0;
2830                 }
2831         }
2832
2833         tw32_f(MAC_TX_AUTO_NEG, 0);
2834
2835         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
2836         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
2837         tw32_f(MAC_MODE, tp->mac_mode);
2838         udelay(40);
2839
2840         if (tp->phy_id == PHY_ID_BCM8002)
2841                 tg3_init_bcm8002(tp);
2842
2843         /* Enable link change event even when serdes polling.  */
2844         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2845         udelay(40);
2846
2847         current_link_up = 0;
2848         mac_status = tr32(MAC_STATUS);
2849
2850         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
2851                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
2852         else
2853                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
2854
2855         tp->hw_status->status =
2856                 (SD_STATUS_UPDATED |
2857                  (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
2858
2859         for (i = 0; i < 100; i++) {
2860                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2861                                     MAC_STATUS_CFG_CHANGED));
2862                 udelay(5);
2863                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
2864                                          MAC_STATUS_CFG_CHANGED |
2865                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
2866                         break;
2867         }
2868
2869         mac_status = tr32(MAC_STATUS);
2870         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
2871                 current_link_up = 0;
2872                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
2873                     tp->serdes_counter == 0) {
2874                         tw32_f(MAC_MODE, (tp->mac_mode |
2875                                           MAC_MODE_SEND_CONFIGS));
2876                         udelay(1);
2877                         tw32_f(MAC_MODE, tp->mac_mode);
2878                 }
2879         }
2880
2881         if (current_link_up == 1) {
2882                 tp->link_config.active_speed = SPEED_1000;
2883                 tp->link_config.active_duplex = DUPLEX_FULL;
2884                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2885                                     LED_CTRL_LNKLED_OVERRIDE |
2886                                     LED_CTRL_1000MBPS_ON));
2887         } else {
2888                 tp->link_config.active_speed = SPEED_INVALID;
2889                 tp->link_config.active_duplex = DUPLEX_INVALID;
2890                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2891                                     LED_CTRL_LNKLED_OVERRIDE |
2892                                     LED_CTRL_TRAFFIC_OVERRIDE));
2893         }
2894
2895         if (current_link_up != netif_carrier_ok(tp->dev)) {
2896                 if (current_link_up)
2897                         netif_carrier_on(tp->dev);
2898                 else
2899                         netif_carrier_off(tp->dev);
2900                 tg3_link_report(tp);
2901         } else {
2902                 u32 now_pause_cfg =
2903                         tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2904                                          TG3_FLAG_TX_PAUSE);
2905                 if (orig_pause_cfg != now_pause_cfg ||
2906                     orig_active_speed != tp->link_config.active_speed ||
2907                     orig_active_duplex != tp->link_config.active_duplex)
2908                         tg3_link_report(tp);
2909         }
2910
2911         return 0;
2912 }
2913
2914 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
2915 {
2916         int current_link_up, err = 0;
2917         u32 bmsr, bmcr;
2918         u16 current_speed;
2919         u8 current_duplex;
2920
2921         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2922         tw32_f(MAC_MODE, tp->mac_mode);
2923         udelay(40);
2924
2925         tw32(MAC_EVENT, 0);
2926
2927         tw32_f(MAC_STATUS,
2928              (MAC_STATUS_SYNC_CHANGED |
2929               MAC_STATUS_CFG_CHANGED |
2930               MAC_STATUS_MI_COMPLETION |
2931               MAC_STATUS_LNKSTATE_CHANGED));
2932         udelay(40);
2933
2934         if (force_reset)
2935                 tg3_phy_reset(tp);
2936
2937         current_link_up = 0;
2938         current_speed = SPEED_INVALID;
2939         current_duplex = DUPLEX_INVALID;
2940
2941         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2942         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2943         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
2944                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2945                         bmsr |= BMSR_LSTATUS;
2946                 else
2947                         bmsr &= ~BMSR_LSTATUS;
2948         }
2949
2950         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
2951
2952         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
2953             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2954                 /* do nothing, just check for link up at the end */
2955         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2956                 u32 adv, new_adv;
2957
2958                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2959                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
2960                                   ADVERTISE_1000XPAUSE |
2961                                   ADVERTISE_1000XPSE_ASYM |
2962                                   ADVERTISE_SLCT);
2963
2964                 /* Always advertise symmetric PAUSE just like copper */
2965                 new_adv |= ADVERTISE_1000XPAUSE;
2966
2967                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2968                         new_adv |= ADVERTISE_1000XHALF;
2969                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2970                         new_adv |= ADVERTISE_1000XFULL;
2971
2972                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
2973                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2974                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
2975                         tg3_writephy(tp, MII_BMCR, bmcr);
2976
2977                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2978                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
2979                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2980
2981                         return err;
2982                 }
2983         } else {
2984                 u32 new_bmcr;
2985
2986                 bmcr &= ~BMCR_SPEED1000;
2987                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
2988
2989                 if (tp->link_config.duplex == DUPLEX_FULL)
2990                         new_bmcr |= BMCR_FULLDPLX;
2991
2992                 if (new_bmcr != bmcr) {
2993                         /* BMCR_SPEED1000 is a reserved bit that needs
2994                          * to be set on write.
2995                          */
2996                         new_bmcr |= BMCR_SPEED1000;
2997
2998                         /* Force a linkdown */
2999                         if (netif_carrier_ok(tp->dev)) {
3000                                 u32 adv;
3001
3002                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
3003                                 adv &= ~(ADVERTISE_1000XFULL |
3004                                          ADVERTISE_1000XHALF |
3005                                          ADVERTISE_SLCT);
3006                                 tg3_writephy(tp, MII_ADVERTISE, adv);
3007                                 tg3_writephy(tp, MII_BMCR, bmcr |
3008                                                            BMCR_ANRESTART |
3009                                                            BMCR_ANENABLE);
3010                                 udelay(10);
3011                                 netif_carrier_off(tp->dev);
3012                         }
3013                         tg3_writephy(tp, MII_BMCR, new_bmcr);
3014                         bmcr = new_bmcr;
3015                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3016                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3017                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
3018                             ASIC_REV_5714) {
3019                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
3020                                         bmsr |= BMSR_LSTATUS;
3021                                 else
3022                                         bmsr &= ~BMSR_LSTATUS;
3023                         }
3024                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3025                 }
3026         }
3027
3028         if (bmsr & BMSR_LSTATUS) {
3029                 current_speed = SPEED_1000;
3030                 current_link_up = 1;
3031                 if (bmcr & BMCR_FULLDPLX)
3032                         current_duplex = DUPLEX_FULL;
3033                 else
3034                         current_duplex = DUPLEX_HALF;
3035
3036                 if (bmcr & BMCR_ANENABLE) {
3037                         u32 local_adv, remote_adv, common;
3038
3039                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
3040                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
3041                         common = local_adv & remote_adv;
3042                         if (common & (ADVERTISE_1000XHALF |
3043                                       ADVERTISE_1000XFULL)) {
3044                                 if (common & ADVERTISE_1000XFULL)
3045                                         current_duplex = DUPLEX_FULL;
3046                                 else
3047                                         current_duplex = DUPLEX_HALF;
3048
3049                                 tg3_setup_flow_control(tp, local_adv,
3050                                                        remote_adv);
3051                         }
3052                         else
3053                                 current_link_up = 0;
3054                 }
3055         }
3056
3057         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
3058         if (tp->link_config.active_duplex == DUPLEX_HALF)
3059                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
3060
3061         tw32_f(MAC_MODE, tp->mac_mode);
3062         udelay(40);
3063
3064         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3065
3066         tp->link_config.active_speed = current_speed;
3067         tp->link_config.active_duplex = current_duplex;
3068
3069         if (current_link_up != netif_carrier_ok(tp->dev)) {
3070                 if (current_link_up)
3071                         netif_carrier_on(tp->dev);
3072                 else {
3073                         netif_carrier_off(tp->dev);
3074                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3075                 }
3076                 tg3_link_report(tp);
3077         }
3078         return err;
3079 }
3080
3081 static void tg3_serdes_parallel_detect(struct tg3 *tp)
3082 {
3083         if (tp->serdes_counter) {
3084                 /* Give autoneg time to complete. */
3085                 tp->serdes_counter--;
3086                 return;
3087         }
3088         if (!netif_carrier_ok(tp->dev) &&
3089             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
3090                 u32 bmcr;
3091
3092                 tg3_readphy(tp, MII_BMCR, &bmcr);
3093                 if (bmcr & BMCR_ANENABLE) {
3094                         u32 phy1, phy2;
3095
3096                         /* Select shadow register 0x1f */
3097                         tg3_writephy(tp, 0x1c, 0x7c00);
3098                         tg3_readphy(tp, 0x1c, &phy1);
3099
3100                         /* Select expansion interrupt status register */
3101                         tg3_writephy(tp, 0x17, 0x0f01);
3102                         tg3_readphy(tp, 0x15, &phy2);
3103                         tg3_readphy(tp, 0x15, &phy2);
3104
3105                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
3106                                 /* We have signal detect and not receiving
3107                                  * config code words, link is up by parallel
3108                                  * detection.
3109                                  */
3110
3111                                 bmcr &= ~BMCR_ANENABLE;
3112                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
3113                                 tg3_writephy(tp, MII_BMCR, bmcr);
3114                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
3115                         }
3116                 }
3117         }
3118         else if (netif_carrier_ok(tp->dev) &&
3119                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
3120                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
3121                 u32 phy2;
3122
3123                 /* Select expansion interrupt status register */
3124                 tg3_writephy(tp, 0x17, 0x0f01);
3125                 tg3_readphy(tp, 0x15, &phy2);
3126                 if (phy2 & 0x20) {
3127                         u32 bmcr;
3128
3129                         /* Config code words received, turn on autoneg. */
3130                         tg3_readphy(tp, MII_BMCR, &bmcr);
3131                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
3132
3133                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3134
3135                 }
3136         }
3137 }
3138
3139 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
3140 {
3141         int err;
3142
3143         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
3144                 err = tg3_setup_fiber_phy(tp, force_reset);
3145         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
3146                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
3147         } else {
3148                 err = tg3_setup_copper_phy(tp, force_reset);
3149         }
3150
3151         if (tp->link_config.active_speed == SPEED_1000 &&
3152             tp->link_config.active_duplex == DUPLEX_HALF)
3153                 tw32(MAC_TX_LENGTHS,
3154                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
3155                       (6 << TX_LENGTHS_IPG_SHIFT) |
3156                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
3157         else
3158                 tw32(MAC_TX_LENGTHS,
3159                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
3160                       (6 << TX_LENGTHS_IPG_SHIFT) |
3161                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
3162
3163         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
3164                 if (netif_carrier_ok(tp->dev)) {
3165                         tw32(HOSTCC_STAT_COAL_TICKS,
3166                              tp->coal.stats_block_coalesce_usecs);
3167                 } else {
3168                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
3169                 }
3170         }
3171
3172         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND) {
3173                 u32 val = tr32(PCIE_PWR_MGMT_THRESH);
3174                 if (!netif_carrier_ok(tp->dev))
3175                         val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
3176                               tp->pwrmgmt_thresh;
3177                 else
3178                         val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
3179                 tw32(PCIE_PWR_MGMT_THRESH, val);
3180         }
3181
3182         return err;
3183 }
3184
3185 /* This is called whenever we suspect that the system chipset is re-
3186  * ordering the sequence of MMIO to the tx send mailbox. The symptom
3187  * is bogus tx completions. We try to recover by setting the
3188  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
3189  * in the workqueue.
3190  */
3191 static void tg3_tx_recover(struct tg3 *tp)
3192 {
3193         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
3194                tp->write32_tx_mbox == tg3_write_indirect_mbox);
3195
3196         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
3197                "mapped I/O cycles to the network device, attempting to "
3198                "recover. Please report the problem to the driver maintainer "
3199                "and include system chipset information.\n", tp->dev->name);
3200
3201         spin_lock(&tp->lock);
3202         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
3203         spin_unlock(&tp->lock);
3204 }
3205
3206 static inline u32 tg3_tx_avail(struct tg3 *tp)
3207 {
3208         smp_mb();
3209         return (tp->tx_pending -
3210                 ((tp->tx_prod - tp->tx_cons) & (TG3_TX_RING_SIZE - 1)));
3211 }
3212
3213 /* Tigon3 never reports partial packet sends.  So we do not
3214  * need special logic to handle SKBs that have not had all
3215  * of their frags sent yet, like SunGEM does.
3216  */
3217 static void tg3_tx(struct tg3 *tp)
3218 {
3219         u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
3220         u32 sw_idx = tp->tx_cons;
3221
3222         while (sw_idx != hw_idx) {
3223                 struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
3224                 struct sk_buff *skb = ri->skb;
3225                 int i, tx_bug = 0;
3226
3227                 if (unlikely(skb == NULL)) {
3228                         tg3_tx_recover(tp);
3229                         return;
3230                 }
3231
3232                 pci_unmap_single(tp->pdev,
3233                                  pci_unmap_addr(ri, mapping),
3234                                  skb_headlen(skb),
3235                                  PCI_DMA_TODEVICE);
3236
3237                 ri->skb = NULL;
3238
3239                 sw_idx = NEXT_TX(sw_idx);
3240
3241                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3242                         ri = &tp->tx_buffers[sw_idx];
3243                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
3244                                 tx_bug = 1;
3245
3246                         pci_unmap_page(tp->pdev,
3247                                        pci_unmap_addr(ri, mapping),
3248                                        skb_shinfo(skb)->frags[i].size,
3249                                        PCI_DMA_TODEVICE);
3250
3251                         sw_idx = NEXT_TX(sw_idx);
3252                 }
3253
3254                 dev_kfree_skb(skb);
3255
3256                 if (unlikely(tx_bug)) {
3257                         tg3_tx_recover(tp);
3258                         return;
3259                 }
3260         }
3261
3262         tp->tx_cons = sw_idx;
3263
3264         /* Need to make the tx_cons update visible to tg3_start_xmit()
3265          * before checking for netif_queue_stopped().  Without the
3266          * memory barrier, there is a small possibility that tg3_start_xmit()
3267          * will miss it and cause the queue to be stopped forever.
3268          */
3269         smp_mb();
3270
3271         if (unlikely(netif_queue_stopped(tp->dev) &&
3272                      (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))) {
3273                 netif_tx_lock(tp->dev);
3274                 if (netif_queue_stopped(tp->dev) &&
3275                     (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))
3276                         netif_wake_queue(tp->dev);
3277                 netif_tx_unlock(tp->dev);
3278         }
3279 }
3280
3281 /* Returns size of skb allocated or < 0 on error.
3282  *
3283  * We only need to fill in the address because the other members
3284  * of the RX descriptor are invariant, see tg3_init_rings.
3285  *
3286  * Note the purposeful assymetry of cpu vs. chip accesses.  For
3287  * posting buffers we only dirty the first cache line of the RX
3288  * descriptor (containing the address).  Whereas for the RX status
3289  * buffers the cpu only reads the last cacheline of the RX descriptor
3290  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
3291  */
3292 static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
3293                             int src_idx, u32 dest_idx_unmasked)
3294 {
3295         struct tg3_rx_buffer_desc *desc;
3296         struct ring_info *map, *src_map;
3297         struct sk_buff *skb;
3298         dma_addr_t mapping;
3299         int skb_size, dest_idx;
3300
3301         src_map = NULL;
3302         switch (opaque_key) {
3303         case RXD_OPAQUE_RING_STD:
3304                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3305                 desc = &tp->rx_std[dest_idx];
3306                 map = &tp->rx_std_buffers[dest_idx];
3307                 if (src_idx >= 0)
3308                         src_map = &tp->rx_std_buffers[src_idx];
3309                 skb_size = tp->rx_pkt_buf_sz;
3310                 break;
3311
3312         case RXD_OPAQUE_RING_JUMBO:
3313                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3314                 desc = &tp->rx_jumbo[dest_idx];
3315                 map = &tp->rx_jumbo_buffers[dest_idx];
3316                 if (src_idx >= 0)
3317                         src_map = &tp->rx_jumbo_buffers[src_idx];
3318                 skb_size = RX_JUMBO_PKT_BUF_SZ;
3319                 break;
3320
3321         default:
3322                 return -EINVAL;
3323         };
3324
3325         /* Do not overwrite any of the map or rp information
3326          * until we are sure we can commit to a new buffer.
3327          *
3328          * Callers depend upon this behavior and assume that
3329          * we leave everything unchanged if we fail.
3330          */
3331         skb = netdev_alloc_skb(tp->dev, skb_size);
3332         if (skb == NULL)
3333                 return -ENOMEM;
3334
3335         skb_reserve(skb, tp->rx_offset);
3336
3337         mapping = pci_map_single(tp->pdev, skb->data,
3338                                  skb_size - tp->rx_offset,
3339                                  PCI_DMA_FROMDEVICE);
3340
3341         map->skb = skb;
3342         pci_unmap_addr_set(map, mapping, mapping);
3343
3344         if (src_map != NULL)
3345                 src_map->skb = NULL;
3346
3347         desc->addr_hi = ((u64)mapping >> 32);
3348         desc->addr_lo = ((u64)mapping & 0xffffffff);
3349
3350         return skb_size;
3351 }
3352
3353 /* We only need to move over in the address because the other
3354  * members of the RX descriptor are invariant.  See notes above
3355  * tg3_alloc_rx_skb for full details.
3356  */
3357 static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
3358                            int src_idx, u32 dest_idx_unmasked)
3359 {
3360         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
3361         struct ring_info *src_map, *dest_map;
3362         int dest_idx;
3363
3364         switch (opaque_key) {
3365         case RXD_OPAQUE_RING_STD:
3366                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3367                 dest_desc = &tp->rx_std[dest_idx];
3368                 dest_map = &tp->rx_std_buffers[dest_idx];
3369                 src_desc = &tp->rx_std[src_idx];
3370                 src_map = &tp->rx_std_buffers[src_idx];
3371                 break;
3372
3373         case RXD_OPAQUE_RING_JUMBO:
3374                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3375                 dest_desc = &tp->rx_jumbo[dest_idx];
3376                 dest_map = &tp->rx_jumbo_buffers[dest_idx];
3377                 src_desc = &tp->rx_jumbo[src_idx];
3378                 src_map = &tp->rx_jumbo_buffers[src_idx];
3379                 break;
3380
3381         default:
3382                 return;
3383         };
3384
3385         dest_map->skb = src_map->skb;
3386         pci_unmap_addr_set(dest_map, mapping,
3387                            pci_unmap_addr(src_map, mapping));
3388         dest_desc->addr_hi = src_desc->addr_hi;
3389         dest_desc->addr_lo = src_desc->addr_lo;
3390
3391         src_map->skb = NULL;
3392 }
3393
3394 #if TG3_VLAN_TAG_USED
3395 static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
3396 {
3397         return vlan_hwaccel_receive_skb(skb, tp->vlgrp, vlan_tag);
3398 }
3399 #endif
3400
3401 /* The RX ring scheme is composed of multiple rings which post fresh
3402  * buffers to the chip, and one special ring the chip uses to report
3403  * status back to the host.
3404  *
3405  * The special ring reports the status of received packets to the
3406  * host.  The chip does not write into the original descriptor the
3407  * RX buffer was obtained from.  The chip simply takes the original
3408  * descriptor as provided by the host, updates the status and length
3409  * field, then writes this into the next status ring entry.
3410  *
3411  * Each ring the host uses to post buffers to the chip is described
3412  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
3413  * it is first placed into the on-chip ram.  When the packet's length
3414  * is known, it walks down the TG3_BDINFO entries to select the ring.
3415  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
3416  * which is within the range of the new packet's length is chosen.
3417  *
3418  * The "separate ring for rx status" scheme may sound queer, but it makes
3419  * sense from a cache coherency perspective.  If only the host writes
3420  * to the buffer post rings, and only the chip writes to the rx status
3421  * rings, then cache lines never move beyond shared-modified state.
3422  * If both the host and chip were to write into the same ring, cache line
3423  * eviction could occur since both entities want it in an exclusive state.
3424  */
3425 static int tg3_rx(struct tg3 *tp, int budget)
3426 {
3427         u32 work_mask, rx_std_posted = 0;
3428         u32 sw_idx = tp->rx_rcb_ptr;
3429         u16 hw_idx;
3430         int received;
3431
3432         hw_idx = tp->hw_status->idx[0].rx_producer;
3433         /*
3434          * We need to order the read of hw_idx and the read of
3435          * the opaque cookie.
3436          */
3437         rmb();
3438         work_mask = 0;
3439         received = 0;
3440         while (sw_idx != hw_idx && budget > 0) {
3441                 struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
3442                 unsigned int len;
3443                 struct sk_buff *skb;
3444                 dma_addr_t dma_addr;
3445                 u32 opaque_key, desc_idx, *post_ptr;
3446
3447                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
3448                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
3449                 if (opaque_key == RXD_OPAQUE_RING_STD) {
3450                         dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
3451                                                   mapping);
3452                         skb = tp->rx_std_buffers[desc_idx].skb;
3453                         post_ptr = &tp->rx_std_ptr;
3454                         rx_std_posted++;
3455                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
3456                         dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
3457                                                   mapping);
3458                         skb = tp->rx_jumbo_buffers[desc_idx].skb;
3459                         post_ptr = &tp->rx_jumbo_ptr;
3460                 }
3461                 else {
3462                         goto next_pkt_nopost;
3463                 }
3464
3465                 work_mask |= opaque_key;
3466
3467                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
3468                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
3469                 drop_it:
3470                         tg3_recycle_rx(tp, opaque_key,
3471                                        desc_idx, *post_ptr);
3472                 drop_it_no_recycle:
3473                         /* Other statistics kept track of by card. */
3474                         tp->net_stats.rx_dropped++;
3475                         goto next_pkt;
3476                 }
3477
3478                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4; /* omit crc */
3479
3480                 if (len > RX_COPY_THRESHOLD
3481                         && tp->rx_offset == 2
3482                         /* rx_offset != 2 iff this is a 5701 card running
3483                          * in PCI-X mode [see tg3_get_invariants()] */
3484                 ) {
3485                         int skb_size;
3486
3487                         skb_size = tg3_alloc_rx_skb(tp, opaque_key,
3488                                                     desc_idx, *post_ptr);
3489                         if (skb_size < 0)
3490                                 goto drop_it;
3491
3492                         pci_unmap_single(tp->pdev, dma_addr,
3493                                          skb_size - tp->rx_offset,
3494                                          PCI_DMA_FROMDEVICE);
3495
3496                         skb_put(skb, len);
3497                 } else {
3498                         struct sk_buff *copy_skb;
3499
3500                         tg3_recycle_rx(tp, opaque_key,
3501                                        desc_idx, *post_ptr);
3502
3503                         copy_skb = netdev_alloc_skb(tp->dev, len + 2);
3504                         if (copy_skb == NULL)
3505                                 goto drop_it_no_recycle;
3506
3507                         skb_reserve(copy_skb, 2);
3508                         skb_put(copy_skb, len);
3509                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3510                         skb_copy_from_linear_data(skb, copy_skb->data, len);
3511                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3512
3513                         /* We'll reuse the original ring buffer. */
3514                         skb = copy_skb;
3515                 }
3516
3517                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
3518                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
3519                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
3520                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
3521                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3522                 else
3523                         skb->ip_summed = CHECKSUM_NONE;
3524
3525                 skb->protocol = eth_type_trans(skb, tp->dev);
3526 #if TG3_VLAN_TAG_USED
3527                 if (tp->vlgrp != NULL &&
3528                     desc->type_flags & RXD_FLAG_VLAN) {
3529                         tg3_vlan_rx(tp, skb,
3530                                     desc->err_vlan & RXD_VLAN_MASK);
3531                 } else
3532 #endif
3533                         netif_receive_skb(skb);
3534
3535                 tp->dev->last_rx = jiffies;
3536                 received++;
3537                 budget--;
3538
3539 next_pkt:
3540                 (*post_ptr)++;
3541
3542                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
3543                         u32 idx = *post_ptr % TG3_RX_RING_SIZE;
3544
3545                         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX +
3546                                      TG3_64BIT_REG_LOW, idx);
3547                         work_mask &= ~RXD_OPAQUE_RING_STD;
3548                         rx_std_posted = 0;
3549                 }
3550 next_pkt_nopost:
3551                 sw_idx++;
3552                 sw_idx &= (TG3_RX_RCB_RING_SIZE(tp) - 1);
3553
3554                 /* Refresh hw_idx to see if there is new work */
3555                 if (sw_idx == hw_idx) {
3556                         hw_idx = tp->hw_status->idx[0].rx_producer;
3557                         rmb();
3558                 }
3559         }
3560
3561         /* ACK the status ring. */
3562         tp->rx_rcb_ptr = sw_idx;
3563         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, sw_idx);
3564
3565         /* Refill RX ring(s). */
3566         if (work_mask & RXD_OPAQUE_RING_STD) {
3567                 sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
3568                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
3569                              sw_idx);
3570         }
3571         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
3572                 sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
3573                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
3574                              sw_idx);
3575         }
3576         mmiowb();
3577
3578         return received;
3579 }
3580
3581 static int tg3_poll_work(struct tg3 *tp, int work_done, int budget)
3582 {
3583         struct tg3_hw_status *sblk = tp->hw_status;
3584
3585         /* handle link change and other phy events */
3586         if (!(tp->tg3_flags &
3587               (TG3_FLAG_USE_LINKCHG_REG |
3588                TG3_FLAG_POLL_SERDES))) {
3589                 if (sblk->status & SD_STATUS_LINK_CHG) {
3590                         sblk->status = SD_STATUS_UPDATED |
3591                                 (sblk->status & ~SD_STATUS_LINK_CHG);
3592                         spin_lock(&tp->lock);
3593                         tg3_setup_phy(tp, 0);
3594                         spin_unlock(&tp->lock);
3595                 }
3596         }
3597
3598         /* run TX completion thread */
3599         if (sblk->idx[0].tx_consumer != tp->tx_cons) {
3600                 tg3_tx(tp);
3601                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
3602                         return work_done;
3603         }
3604
3605         /* run RX thread, within the bounds set by NAPI.
3606          * All RX "locking" is done by ensuring outside
3607          * code synchronizes with tg3->napi.poll()
3608          */
3609         if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
3610                 work_done += tg3_rx(tp, budget - work_done);
3611
3612         return work_done;
3613 }
3614
3615 static int tg3_poll(struct napi_struct *napi, int budget)
3616 {
3617         struct tg3 *tp = container_of(napi, struct tg3, napi);
3618         int work_done = 0;
3619         struct tg3_hw_status *sblk = tp->hw_status;
3620
3621         while (1) {
3622                 work_done = tg3_poll_work(tp, work_done, budget);
3623
3624                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
3625                         goto tx_recovery;
3626
3627                 if (unlikely(work_done >= budget))
3628                         break;
3629
3630                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
3631                         /* tp->last_tag is used in tg3_restart_ints() below
3632                          * to tell the hw how much work has been processed,
3633                          * so we must read it before checking for more work.
3634                          */
3635                         tp->last_tag = sblk->status_tag;
3636                         rmb();
3637                 } else
3638                         sblk->status &= ~SD_STATUS_UPDATED;
3639
3640                 if (likely(!tg3_has_work(tp))) {
3641                         netif_rx_complete(tp->dev, napi);
3642                         tg3_restart_ints(tp);
3643                         break;
3644                 }
3645         }
3646
3647         return work_done;
3648
3649 tx_recovery:
3650         /* work_done is guaranteed to be less than budget. */
3651         netif_rx_complete(tp->dev, napi);
3652         schedule_work(&tp->reset_task);
3653         return work_done;
3654 }
3655
3656 static void tg3_irq_quiesce(struct tg3 *tp)
3657 {
3658         BUG_ON(tp->irq_sync);
3659
3660         tp->irq_sync = 1;
3661         smp_mb();
3662
3663         synchronize_irq(tp->pdev->irq);
3664 }
3665
3666 static inline int tg3_irq_sync(struct tg3 *tp)
3667 {
3668         return tp->irq_sync;
3669 }
3670
3671 /* Fully shutdown all tg3 driver activity elsewhere in the system.
3672  * If irq_sync is non-zero, then the IRQ handler must be synchronized
3673  * with as well.  Most of the time, this is not necessary except when
3674  * shutting down the device.
3675  */
3676 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
3677 {
3678         spin_lock_bh(&tp->lock);
3679         if (irq_sync)
3680                 tg3_irq_quiesce(tp);
3681 }
3682
3683 static inline void tg3_full_unlock(struct tg3 *tp)
3684 {
3685         spin_unlock_bh(&tp->lock);
3686 }
3687
3688 /* One-shot MSI handler - Chip automatically disables interrupt
3689  * after sending MSI so driver doesn't have to do it.
3690  */
3691 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
3692 {
3693         struct net_device *dev = dev_id;
3694         struct tg3 *tp = netdev_priv(dev);
3695
3696         prefetch(tp->hw_status);
3697         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3698
3699         if (likely(!tg3_irq_sync(tp)))
3700                 netif_rx_schedule(dev, &tp->napi);
3701
3702         return IRQ_HANDLED;
3703 }
3704
3705 /* MSI ISR - No need to check for interrupt sharing and no need to
3706  * flush status block and interrupt mailbox. PCI ordering rules
3707  * guarantee that MSI will arrive after the status block.
3708  */
3709 static irqreturn_t tg3_msi(int irq, void *dev_id)
3710 {
3711         struct net_device *dev = dev_id;
3712         struct tg3 *tp = netdev_priv(dev);
3713
3714         prefetch(tp->hw_status);
3715         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3716         /*
3717          * Writing any value to intr-mbox-0 clears PCI INTA# and
3718          * chip-internal interrupt pending events.
3719          * Writing non-zero to intr-mbox-0 additional tells the
3720          * NIC to stop sending us irqs, engaging "in-intr-handler"
3721          * event coalescing.
3722          */
3723         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3724         if (likely(!tg3_irq_sync(tp)))
3725                 netif_rx_schedule(dev, &tp->napi);
3726
3727         return IRQ_RETVAL(1);
3728 }
3729
3730 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
3731 {
3732         struct net_device *dev = dev_id;
3733         struct tg3 *tp = netdev_priv(dev);
3734         struct tg3_hw_status *sblk = tp->hw_status;
3735         unsigned int handled = 1;
3736
3737         /* In INTx mode, it is possible for the interrupt to arrive at
3738          * the CPU before the status block posted prior to the interrupt.
3739          * Reading the PCI State register will confirm whether the
3740          * interrupt is ours and will flush the status block.
3741          */
3742         if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
3743                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
3744                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3745                         handled = 0;
3746                         goto out;
3747                 }
3748         }
3749
3750         /*
3751          * Writing any value to intr-mbox-0 clears PCI INTA# and
3752          * chip-internal interrupt pending events.
3753          * Writing non-zero to intr-mbox-0 additional tells the
3754          * NIC to stop sending us irqs, engaging "in-intr-handler"
3755          * event coalescing.
3756          *
3757          * Flush the mailbox to de-assert the IRQ immediately to prevent
3758          * spurious interrupts.  The flush impacts performance but
3759          * excessive spurious interrupts can be worse in some cases.
3760          */
3761         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3762         if (tg3_irq_sync(tp))
3763                 goto out;
3764         sblk->status &= ~SD_STATUS_UPDATED;
3765         if (likely(tg3_has_work(tp))) {
3766                 prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3767                 netif_rx_schedule(dev, &tp->napi);
3768         } else {
3769                 /* No work, shared interrupt perhaps?  re-enable
3770                  * interrupts, and flush that PCI write
3771                  */
3772                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3773                                0x00000000);
3774         }
3775 out:
3776         return IRQ_RETVAL(handled);
3777 }
3778
3779 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
3780 {
3781         struct net_device *dev = dev_id;
3782         struct tg3 *tp = netdev_priv(dev);
3783         struct tg3_hw_status *sblk = tp->hw_status;
3784         unsigned int handled = 1;
3785
3786         /* In INTx mode, it is possible for the interrupt to arrive at
3787          * the CPU before the status block posted prior to the interrupt.
3788          * Reading the PCI State register will confirm whether the
3789          * interrupt is ours and will flush the status block.
3790          */
3791         if (unlikely(sblk->status_tag == tp->last_tag)) {
3792                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
3793                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3794                         handled = 0;
3795                         goto out;
3796                 }
3797         }
3798
3799         /*
3800          * writing any value to intr-mbox-0 clears PCI INTA# and
3801          * chip-internal interrupt pending events.
3802          * writing non-zero to intr-mbox-0 additional tells the
3803          * NIC to stop sending us irqs, engaging "in-intr-handler"
3804          * event coalescing.
3805          *
3806          * Flush the mailbox to de-assert the IRQ immediately to prevent
3807          * spurious interrupts.  The flush impacts performance but
3808          * excessive spurious interrupts can be worse in some cases.
3809          */
3810         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3811         if (tg3_irq_sync(tp))
3812                 goto out;
3813         if (netif_rx_schedule_prep(dev, &tp->napi)) {
3814                 prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3815                 /* Update last_tag to mark that this status has been
3816                  * seen. Because interrupt may be shared, we may be
3817                  * racing with tg3_poll(), so only update last_tag
3818                  * if tg3_poll() is not scheduled.
3819                  */
3820                 tp->last_tag = sblk->status_tag;
3821                 __netif_rx_schedule(dev, &tp->napi);
3822         }
3823 out:
3824         return IRQ_RETVAL(handled);
3825 }
3826
3827 /* ISR for interrupt test */
3828 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
3829 {
3830         struct net_device *dev = dev_id;
3831         struct tg3 *tp = netdev_priv(dev);
3832         struct tg3_hw_status *sblk = tp->hw_status;
3833
3834         if ((sblk->status & SD_STATUS_UPDATED) ||
3835             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3836                 tg3_disable_ints(tp);
3837                 return IRQ_RETVAL(1);
3838         }
3839         return IRQ_RETVAL(0);
3840 }
3841
3842 static int tg3_init_hw(struct tg3 *, int);
3843 static int tg3_halt(struct tg3 *, int, int);
3844
3845 /* Restart hardware after configuration changes, self-test, etc.
3846  * Invoked with tp->lock held.
3847  */
3848 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
3849 {
3850         int err;
3851
3852         err = tg3_init_hw(tp, reset_phy);
3853         if (err) {
3854                 printk(KERN_ERR PFX "%s: Failed to re-initialize device, "
3855                        "aborting.\n", tp->dev->name);
3856                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
3857                 tg3_full_unlock(tp);
3858                 del_timer_sync(&tp->timer);
3859                 tp->irq_sync = 0;
3860                 napi_enable(&tp->napi);
3861                 dev_close(tp->dev);
3862                 tg3_full_lock(tp, 0);
3863         }
3864         return err;
3865 }
3866
3867 #ifdef CONFIG_NET_POLL_CONTROLLER
3868 static void tg3_poll_controller(struct net_device *dev)
3869 {
3870         struct tg3 *tp = netdev_priv(dev);
3871
3872         tg3_interrupt(tp->pdev->irq, dev);
3873 }
3874 #endif
3875
3876 static void tg3_reset_task(struct work_struct *work)
3877 {
3878         struct tg3 *tp = container_of(work, struct tg3, reset_task);
3879         unsigned int restart_timer;
3880
3881         tg3_full_lock(tp, 0);
3882
3883         if (!netif_running(tp->dev)) {
3884                 tg3_full_unlock(tp);
3885                 return;
3886         }
3887
3888         tg3_full_unlock(tp);
3889
3890         tg3_netif_stop(tp);
3891
3892         tg3_full_lock(tp, 1);
3893
3894         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
3895         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
3896
3897         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
3898                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
3899                 tp->write32_rx_mbox = tg3_write_flush_reg32;
3900                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
3901                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
3902         }
3903
3904         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
3905         if (tg3_init_hw(tp, 1))
3906                 goto out;
3907
3908         tg3_netif_start(tp);
3909
3910         if (restart_timer)
3911                 mod_timer(&tp->timer, jiffies + 1);
3912
3913 out:
3914         tg3_full_unlock(tp);
3915 }
3916
3917 static void tg3_dump_short_state(struct tg3 *tp)
3918 {
3919         printk(KERN_ERR PFX "DEBUG: MAC_TX_STATUS[%08x] MAC_RX_STATUS[%08x]\n",
3920                tr32(MAC_TX_STATUS), tr32(MAC_RX_STATUS));
3921         printk(KERN_ERR PFX "DEBUG: RDMAC_STATUS[%08x] WDMAC_STATUS[%08x]\n",
3922                tr32(RDMAC_STATUS), tr32(WDMAC_STATUS));
3923 }
3924
3925 static void tg3_tx_timeout(struct net_device *dev)
3926 {
3927         struct tg3 *tp = netdev_priv(dev);
3928
3929         if (netif_msg_tx_err(tp)) {
3930                 printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
3931                        dev->name);
3932                 tg3_dump_short_state(tp);
3933         }
3934
3935         schedule_work(&tp->reset_task);
3936 }
3937
3938 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
3939 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
3940 {
3941         u32 base = (u32) mapping & 0xffffffff;
3942
3943         return ((base > 0xffffdcc0) &&
3944                 (base + len + 8 < base));
3945 }
3946
3947 /* Test for DMA addresses > 40-bit */
3948 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
3949                                           int len)
3950 {
3951 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
3952         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
3953                 return (((u64) mapping + len) > DMA_40BIT_MASK);
3954         return 0;
3955 #else
3956         return 0;
3957 #endif
3958 }
3959
3960 static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);
3961
3962 /* Workaround 4GB and 40-bit hardware DMA bugs. */
3963 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
3964                                        u32 last_plus_one, u32 *start,
3965                                        u32 base_flags, u32 mss)
3966 {
3967         struct sk_buff *new_skb = skb_copy(skb, GFP_ATOMIC);
3968         dma_addr_t new_addr = 0;
3969         u32 entry = *start;
3970         int i, ret = 0;
3971
3972         if (!new_skb) {
3973                 ret = -1;
3974         } else {
3975                 /* New SKB is guaranteed to be linear. */
3976                 entry = *start;
3977                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
3978                                           PCI_DMA_TODEVICE);
3979                 /* Make sure new skb does not cross any 4G boundaries.
3980                  * Drop the packet if it does.
3981                  */
3982                 if (tg3_4g_overflow_test(new_addr, new_skb->len)) {
3983                         ret = -1;
3984                         dev_kfree_skb(new_skb);
3985                         new_skb = NULL;
3986                 } else {
3987                         tg3_set_txd(tp, entry, new_addr, new_skb->len,
3988                                     base_flags, 1 | (mss << 1));
3989                         *start = NEXT_TX(entry);
3990                 }
3991         }
3992
3993         /* Now clean up the sw ring entries. */
3994         i = 0;
3995         while (entry != last_plus_one) {
3996                 int len;
3997
3998                 if (i == 0)
3999                         len = skb_headlen(skb);
4000                 else
4001                         len = skb_shinfo(skb)->frags[i-1].size;
4002                 pci_unmap_single(tp->pdev,
4003                                  pci_unmap_addr(&tp->tx_buffers[entry], mapping),
4004                                  len, PCI_DMA_TODEVICE);
4005                 if (i == 0) {
4006                         tp->tx_buffers[entry].skb = new_skb;
4007                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, new_addr);
4008                 } else {
4009                         tp->tx_buffers[entry].skb = NULL;
4010                 }
4011                 entry = NEXT_TX(entry);
4012                 i++;
4013         }
4014
4015         dev_kfree_skb(skb);
4016
4017         return ret;
4018 }
4019
4020 static void tg3_set_txd(struct tg3 *tp, int entry,
4021                         dma_addr_t mapping, int len, u32 flags,
4022                         u32 mss_and_is_end)
4023 {
4024         struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];
4025         int is_end = (mss_and_is_end & 0x1);
4026         u32 mss = (mss_and_is_end >> 1);
4027         u32 vlan_tag = 0;
4028
4029         if (is_end)
4030                 flags |= TXD_FLAG_END;
4031         if (flags & TXD_FLAG_VLAN) {
4032                 vlan_tag = flags >> 16;
4033                 flags &= 0xffff;
4034         }
4035         vlan_tag |= (mss << TXD_MSS_SHIFT);
4036
4037         txd->addr_hi = ((u64) mapping >> 32);
4038         txd->addr_lo = ((u64) mapping & 0xffffffff);
4039         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
4040         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
4041 }
4042
4043 /* hard_start_xmit for devices that don't have any bugs and
4044  * support TG3_FLG2_HW_TSO_2 only.
4045  */
4046 static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
4047 {
4048         struct tg3 *tp = netdev_priv(dev);
4049         dma_addr_t mapping;
4050         u32 len, entry, base_flags, mss;
4051
4052         len = skb_headlen(skb);
4053
4054         /* We are running in BH disabled context with netif_tx_lock
4055          * and TX reclaim runs via tp->napi.poll inside of a software
4056          * interrupt.  Furthermore, IRQ processing runs lockless so we have
4057          * no IRQ context deadlocks to worry about either.  Rejoice!
4058          */
4059         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
4060                 if (!netif_queue_stopped(dev)) {
4061                         netif_stop_queue(dev);
4062
4063                         /* This is a hard error, log it. */
4064                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
4065                                "queue awake!\n", dev->name);
4066                 }
4067                 return NETDEV_TX_BUSY;
4068         }
4069
4070         entry = tp->tx_prod;
4071         base_flags = 0;
4072         mss = 0;
4073         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
4074                 int tcp_opt_len, ip_tcp_len;
4075
4076                 if (skb_header_cloned(skb) &&
4077                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
4078                         dev_kfree_skb(skb);
4079                         goto out_unlock;
4080                 }
4081
4082                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
4083                         mss |= (skb_headlen(skb) - ETH_HLEN) << 9;
4084                 else {
4085                         struct iphdr *iph = ip_hdr(skb);
4086
4087                         tcp_opt_len = tcp_optlen(skb);
4088                         ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
4089
4090                         iph->check = 0;
4091                         iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
4092                         mss |= (ip_tcp_len + tcp_opt_len) << 9;
4093                 }
4094
4095                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
4096                                TXD_FLAG_CPU_POST_DMA);
4097
4098                 tcp_hdr(skb)->check = 0;
4099
4100         }
4101         else if (skb->ip_summed == CHECKSUM_PARTIAL)
4102                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
4103 #if TG3_VLAN_TAG_USED
4104         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
4105                 base_flags |= (TXD_FLAG_VLAN |
4106                                (vlan_tx_tag_get(skb) << 16));
4107 #endif
4108
4109         /* Queue skb data, a.k.a. the main skb fragment. */
4110         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
4111
4112         tp->tx_buffers[entry].skb = skb;
4113         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4114
4115         tg3_set_txd(tp, entry, mapping, len, base_flags,
4116                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
4117
4118         entry = NEXT_TX(entry);
4119
4120         /* Now loop through additional data fragments, and queue them. */
4121         if (skb_shinfo(skb)->nr_frags > 0) {
4122                 unsigned int i, last;
4123
4124                 last = skb_shinfo(skb)->nr_frags - 1;
4125                 for (i = 0; i <= last; i++) {
4126                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4127
4128                         len = frag->size;
4129                         mapping = pci_map_page(tp->pdev,
4130                                                frag->page,
4131                                                frag->page_offset,
4132                                                len, PCI_DMA_TODEVICE);
4133
4134                         tp->tx_buffers[entry].skb = NULL;
4135                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4136
4137                         tg3_set_txd(tp, entry, mapping, len,
4138                                     base_flags, (i == last) | (mss << 1));
4139
4140                         entry = NEXT_TX(entry);
4141                 }
4142         }
4143
4144         /* Packets are ready, update Tx producer idx local and on card. */
4145         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4146
4147         tp->tx_prod = entry;
4148         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
4149                 netif_stop_queue(dev);
4150                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
4151                         netif_wake_queue(tp->dev);
4152         }
4153
4154 out_unlock:
4155         mmiowb();
4156
4157         dev->trans_start = jiffies;
4158
4159         return NETDEV_TX_OK;
4160 }
4161
4162 static int tg3_start_xmit_dma_bug(struct sk_buff *, struct net_device *);
4163
4164 /* Use GSO to workaround a rare TSO bug that may be triggered when the
4165  * TSO header is greater than 80 bytes.
4166  */
4167 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
4168 {
4169         struct sk_buff *segs, *nskb;
4170
4171         /* Estimate the number of fragments in the worst case */
4172         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))) {
4173                 netif_stop_queue(tp->dev);
4174                 if (tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))
4175                         return NETDEV_TX_BUSY;
4176
4177                 netif_wake_queue(tp->dev);
4178         }
4179
4180         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
4181         if (unlikely(IS_ERR(segs)))
4182                 goto tg3_tso_bug_end;
4183
4184         do {
4185                 nskb = segs;
4186                 segs = segs->next;
4187                 nskb->next = NULL;
4188                 tg3_start_xmit_dma_bug(nskb, tp->dev);
4189         } while (segs);
4190
4191 tg3_tso_bug_end:
4192         dev_kfree_skb(skb);
4193
4194         return NETDEV_TX_OK;
4195 }
4196
4197 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
4198  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
4199  */
4200 static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
4201 {
4202         struct tg3 *tp = netdev_priv(dev);
4203         dma_addr_t mapping;
4204         u32 len, entry, base_flags, mss;
4205         int would_hit_hwbug;
4206
4207         len = skb_headlen(skb);
4208
4209         /* We are running in BH disabled context with netif_tx_lock
4210          * and TX reclaim runs via tp->napi.poll inside of a software
4211          * interrupt.  Furthermore, IRQ processing runs lockless so we have
4212          * no IRQ context deadlocks to worry about either.  Rejoice!
4213          */
4214         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
4215                 if (!netif_queue_stopped(dev)) {
4216                         netif_stop_queue(dev);
4217
4218                         /* This is a hard error, log it. */
4219                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
4220                                "queue awake!\n", dev->name);
4221                 }
4222                 return NETDEV_TX_BUSY;
4223         }
4224
4225         entry = tp->tx_prod;
4226         base_flags = 0;
4227         if (skb->ip_summed == CHECKSUM_PARTIAL)
4228                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
4229         mss = 0;
4230         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
4231                 struct iphdr *iph;
4232                 int tcp_opt_len, ip_tcp_len, hdr_len;
4233
4234                 if (skb_header_cloned(skb) &&
4235                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
4236                         dev_kfree_skb(skb);
4237                         goto out_unlock;
4238                 }
4239
4240                 tcp_opt_len = tcp_optlen(skb);
4241                 ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
4242
4243                 hdr_len = ip_tcp_len + tcp_opt_len;
4244                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
4245                              (tp->tg3_flags2 & TG3_FLG2_TSO_BUG))
4246                         return (tg3_tso_bug(tp, skb));
4247
4248                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
4249                                TXD_FLAG_CPU_POST_DMA);
4250
4251                 iph = ip_hdr(skb);
4252                 iph->check = 0;
4253                 iph->tot_len = htons(mss + hdr_len);
4254                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
4255                         tcp_hdr(skb)->check = 0;
4256                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
4257                 } else
4258                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4259                                                                  iph->daddr, 0,
4260                                                                  IPPROTO_TCP,
4261                                                                  0);
4262
4263                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
4264                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
4265                         if (tcp_opt_len || iph->ihl > 5) {
4266                                 int tsflags;
4267
4268                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
4269                                 mss |= (tsflags << 11);
4270                         }
4271                 } else {
4272                         if (tcp_opt_len || iph->ihl > 5) {
4273                                 int tsflags;
4274
4275                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
4276                                 base_flags |= tsflags << 12;
4277                         }
4278                 }
4279         }
4280 #if TG3_VLAN_TAG_USED
4281         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
4282                 base_flags |= (TXD_FLAG_VLAN |
4283                                (vlan_tx_tag_get(skb) << 16));
4284 #endif
4285
4286         /* Queue skb data, a.k.a. the main skb fragment. */
4287         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
4288
4289         tp->tx_buffers[entry].skb = skb;
4290         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4291
4292         would_hit_hwbug = 0;
4293
4294         if (tg3_4g_overflow_test(mapping, len))
4295                 would_hit_hwbug = 1;
4296
4297         tg3_set_txd(tp, entry, mapping, len, base_flags,
4298                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
4299
4300         entry = NEXT_TX(entry);
4301
4302         /* Now loop through additional data fragments, and queue them. */
4303         if (skb_shinfo(skb)->nr_frags > 0) {
4304                 unsigned int i, last;
4305
4306                 last = skb_shinfo(skb)->nr_frags - 1;
4307                 for (i = 0; i <= last; i++) {
4308                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4309
4310                         len = frag->size;
4311                         mapping = pci_map_page(tp->pdev,
4312                                                frag->page,
4313                                                frag->page_offset,
4314                                                len, PCI_DMA_TODEVICE);
4315
4316                         tp->tx_buffers[entry].skb = NULL;
4317                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4318
4319                         if (tg3_4g_overflow_test(mapping, len))
4320                                 would_hit_hwbug = 1;
4321
4322                         if (tg3_40bit_overflow_test(tp, mapping, len))
4323                                 would_hit_hwbug = 1;
4324
4325                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
4326                                 tg3_set_txd(tp, entry, mapping, len,
4327                                             base_flags, (i == last)|(mss << 1));
4328                         else
4329                                 tg3_set_txd(tp, entry, mapping, len,
4330                                             base_flags, (i == last));
4331
4332                         entry = NEXT_TX(entry);
4333                 }
4334         }
4335
4336         if (would_hit_hwbug) {
4337                 u32 last_plus_one = entry;
4338                 u32 start;
4339
4340                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
4341                 start &= (TG3_TX_RING_SIZE - 1);
4342
4343                 /* If the workaround fails due to memory/mapping
4344                  * failure, silently drop this packet.
4345                  */
4346                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
4347                                                 &start, base_flags, mss))
4348                         goto out_unlock;
4349
4350                 entry = start;
4351         }
4352
4353         /* Packets are ready, update Tx producer idx local and on card. */
4354         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4355
4356         tp->tx_prod = entry;
4357         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
4358                 netif_stop_queue(dev);
4359                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
4360                         netif_wake_queue(tp->dev);
4361         }
4362
4363 out_unlock:
4364         mmiowb();
4365
4366         dev->trans_start = jiffies;
4367
4368         return NETDEV_TX_OK;
4369 }
4370
4371 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
4372                                int new_mtu)
4373 {
4374         dev->mtu = new_mtu;
4375
4376         if (new_mtu > ETH_DATA_LEN) {
4377                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4378                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
4379                         ethtool_op_set_tso(dev, 0);
4380                 }
4381                 else
4382                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
4383         } else {
4384                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
4385                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
4386                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
4387         }
4388 }
4389
4390 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
4391 {
4392         struct tg3 *tp = netdev_priv(dev);
4393         int err;
4394
4395         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
4396                 return -EINVAL;
4397
4398         if (!netif_running(dev)) {
4399                 /* We'll just catch it later when the
4400                  * device is up'd.
4401                  */
4402                 tg3_set_mtu(dev, tp, new_mtu);
4403                 return 0;
4404         }
4405
4406         tg3_netif_stop(tp);
4407
4408         tg3_full_lock(tp, 1);
4409
4410         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4411
4412         tg3_set_mtu(dev, tp, new_mtu);
4413
4414         err = tg3_restart_hw(tp, 0);
4415
4416         if (!err)
4417                 tg3_netif_start(tp);
4418
4419         tg3_full_unlock(tp);
4420
4421         return err;
4422 }
4423
4424 /* Free up pending packets in all rx/tx rings.
4425  *
4426  * The chip has been shut down and the driver detached from
4427  * the networking, so no interrupts or new tx packets will
4428  * end up in the driver.  tp->{tx,}lock is not held and we are not
4429  * in an interrupt context and thus may sleep.
4430  */
4431 static void tg3_free_rings(struct tg3 *tp)
4432 {
4433         struct ring_info *rxp;
4434         int i;
4435
4436         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4437                 rxp = &tp->rx_std_buffers[i];
4438
4439                 if (rxp->skb == NULL)
4440                         continue;
4441                 pci_unmap_single(tp->pdev,
4442                                  pci_unmap_addr(rxp, mapping),
4443                                  tp->rx_pkt_buf_sz - tp->rx_offset,
4444                                  PCI_DMA_FROMDEVICE);
4445                 dev_kfree_skb_any(rxp->skb);
4446                 rxp->skb = NULL;
4447         }
4448
4449         for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4450                 rxp = &tp->rx_jumbo_buffers[i];
4451
4452                 if (rxp->skb == NULL)
4453                         continue;
4454                 pci_unmap_single(tp->pdev,
4455                                  pci_unmap_addr(rxp, mapping),
4456                                  RX_JUMBO_PKT_BUF_SZ - tp->rx_offset,
4457                                  PCI_DMA_FROMDEVICE);
4458                 dev_kfree_skb_any(rxp->skb);
4459                 rxp->skb = NULL;
4460         }
4461
4462         for (i = 0; i < TG3_TX_RING_SIZE; ) {
4463                 struct tx_ring_info *txp;
4464                 struct sk_buff *skb;
4465                 int j;
4466
4467                 txp = &tp->tx_buffers[i];
4468                 skb = txp->skb;
4469
4470                 if (skb == NULL) {
4471                         i++;
4472                         continue;
4473                 }
4474
4475                 pci_unmap_single(tp->pdev,
4476                                  pci_unmap_addr(txp, mapping),
4477                                  skb_headlen(skb),
4478                                  PCI_DMA_TODEVICE);
4479                 txp->skb = NULL;
4480
4481                 i++;
4482
4483                 for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) {
4484                         txp = &tp->tx_buffers[i & (TG3_TX_RING_SIZE - 1)];
4485                         pci_unmap_page(tp->pdev,
4486                                        pci_unmap_addr(txp, mapping),
4487                                        skb_shinfo(skb)->frags[j].size,
4488                                        PCI_DMA_TODEVICE);
4489                         i++;
4490                 }
4491
4492                 dev_kfree_skb_any(skb);
4493         }
4494 }
4495
4496 /* Initialize tx/rx rings for packet processing.
4497  *
4498  * The chip has been shut down and the driver detached from
4499  * the networking, so no interrupts or new tx packets will
4500  * end up in the driver.  tp->{tx,}lock are held and thus
4501  * we may not sleep.
4502  */
4503 static int tg3_init_rings(struct tg3 *tp)
4504 {
4505         u32 i;
4506
4507         /* Free up all the SKBs. */
4508         tg3_free_rings(tp);
4509
4510         /* Zero out all descriptors. */
4511         memset(tp->rx_std, 0, TG3_RX_RING_BYTES);
4512         memset(tp->rx_jumbo, 0, TG3_RX_JUMBO_RING_BYTES);
4513         memset(tp->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
4514         memset(tp->tx_ring, 0, TG3_TX_RING_BYTES);
4515
4516         tp->rx_pkt_buf_sz = RX_PKT_BUF_SZ;
4517         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
4518             (tp->dev->mtu > ETH_DATA_LEN))
4519                 tp->rx_pkt_buf_sz = RX_JUMBO_PKT_BUF_SZ;
4520
4521         /* Initialize invariants of the rings, we only set this
4522          * stuff once.  This works because the card does not
4523          * write into the rx buffer posting rings.
4524          */
4525         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4526                 struct tg3_rx_buffer_desc *rxd;
4527
4528                 rxd = &tp->rx_std[i];
4529                 rxd->idx_len = (tp->rx_pkt_buf_sz - tp->rx_offset - 64)
4530                         << RXD_LEN_SHIFT;
4531                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
4532                 rxd->opaque = (RXD_OPAQUE_RING_STD |
4533                                (i << RXD_OPAQUE_INDEX_SHIFT));
4534         }
4535
4536         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4537                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4538                         struct tg3_rx_buffer_desc *rxd;
4539
4540                         rxd = &tp->rx_jumbo[i];
4541                         rxd->idx_len = (RX_JUMBO_PKT_BUF_SZ - tp->rx_offset - 64)
4542                                 << RXD_LEN_SHIFT;
4543                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
4544                                 RXD_FLAG_JUMBO;
4545                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
4546                                (i << RXD_OPAQUE_INDEX_SHIFT));
4547                 }
4548         }
4549
4550         /* Now allocate fresh SKBs for each rx ring. */
4551         for (i = 0; i < tp->rx_pending; i++) {
4552                 if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_STD, -1, i) < 0) {
4553                         printk(KERN_WARNING PFX
4554                                "%s: Using a smaller RX standard ring, "
4555                                "only %d out of %d buffers were allocated "
4556                                "successfully.\n",
4557                                tp->dev->name, i, tp->rx_pending);
4558                         if (i == 0)
4559                                 return -ENOMEM;
4560                         tp->rx_pending = i;
4561                         break;
4562                 }
4563         }
4564
4565         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4566                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
4567                         if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_JUMBO,
4568                                              -1, i) < 0) {
4569                                 printk(KERN_WARNING PFX
4570                                        "%s: Using a smaller RX jumbo ring, "
4571                                        "only %d out of %d buffers were "
4572                                        "allocated successfully.\n",
4573                                        tp->dev->name, i, tp->rx_jumbo_pending);
4574                                 if (i == 0) {
4575                                         tg3_free_rings(tp);
4576                                         return -ENOMEM;
4577                                 }
4578                                 tp->rx_jumbo_pending = i;
4579                                 break;
4580                         }
4581                 }
4582         }
4583         return 0;
4584 }
4585
4586 /*
4587  * Must not be invoked with interrupt sources disabled and
4588  * the hardware shutdown down.
4589  */
4590 static void tg3_free_consistent(struct tg3 *tp)
4591 {
4592         kfree(tp->rx_std_buffers);
4593         tp->rx_std_buffers = NULL;
4594         if (tp->rx_std) {
4595                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
4596                                     tp->rx_std, tp->rx_std_mapping);
4597                 tp->rx_std = NULL;
4598         }
4599         if (tp->rx_jumbo) {
4600                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4601                                     tp->rx_jumbo, tp->rx_jumbo_mapping);
4602                 tp->rx_jumbo = NULL;
4603         }
4604         if (tp->rx_rcb) {
4605                 pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4606                                     tp->rx_rcb, tp->rx_rcb_mapping);
4607                 tp->rx_rcb = NULL;
4608         }
4609         if (tp->tx_ring) {
4610                 pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
4611                         tp->tx_ring, tp->tx_desc_mapping);
4612                 tp->tx_ring = NULL;
4613         }
4614         if (tp->hw_status) {
4615                 pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
4616                                     tp->hw_status, tp->status_mapping);
4617                 tp->hw_status = NULL;
4618         }
4619         if (tp->hw_stats) {
4620                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
4621                                     tp->hw_stats, tp->stats_mapping);
4622                 tp->hw_stats = NULL;
4623         }
4624 }
4625
4626 /*
4627  * Must not be invoked with interrupt sources disabled and
4628  * the hardware shutdown down.  Can sleep.
4629  */
4630 static int tg3_alloc_consistent(struct tg3 *tp)
4631 {
4632         tp->rx_std_buffers = kzalloc((sizeof(struct ring_info) *
4633                                       (TG3_RX_RING_SIZE +
4634                                        TG3_RX_JUMBO_RING_SIZE)) +
4635                                      (sizeof(struct tx_ring_info) *
4636                                       TG3_TX_RING_SIZE),
4637                                      GFP_KERNEL);
4638         if (!tp->rx_std_buffers)
4639                 return -ENOMEM;
4640
4641         tp->rx_jumbo_buffers = &tp->rx_std_buffers[TG3_RX_RING_SIZE];
4642         tp->tx_buffers = (struct tx_ring_info *)
4643                 &tp->rx_jumbo_buffers[TG3_RX_JUMBO_RING_SIZE];
4644
4645         tp->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
4646                                           &tp->rx_std_mapping);
4647         if (!tp->rx_std)
4648                 goto err_out;
4649
4650         tp->rx_jumbo = pci_alloc_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4651                                             &tp->rx_jumbo_mapping);
4652
4653         if (!tp->rx_jumbo)
4654                 goto err_out;
4655
4656         tp->rx_rcb = pci_alloc_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4657                                           &tp->rx_rcb_mapping);
4658         if (!tp->rx_rcb)
4659                 goto err_out;
4660
4661         tp->tx_ring = pci_alloc_consistent(tp->pdev, TG3_TX_RING_BYTES,
4662                                            &tp->tx_desc_mapping);
4663         if (!tp->tx_ring)
4664                 goto err_out;
4665
4666         tp->hw_status = pci_alloc_consistent(tp->pdev,
4667                                              TG3_HW_STATUS_SIZE,
4668                                              &tp->status_mapping);
4669         if (!tp->hw_status)
4670                 goto err_out;
4671
4672         tp->hw_stats = pci_alloc_consistent(tp->pdev,
4673                                             sizeof(struct tg3_hw_stats),
4674                                             &tp->stats_mapping);
4675         if (!tp->hw_stats)
4676                 goto err_out;
4677
4678         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4679         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4680
4681         return 0;
4682
4683 err_out:
4684         tg3_free_consistent(tp);
4685         return -ENOMEM;
4686 }
4687
4688 #define MAX_WAIT_CNT 1000
4689
4690 /* To stop a block, clear the enable bit and poll till it
4691  * clears.  tp->lock is held.
4692  */
4693 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
4694 {
4695         unsigned int i;
4696         u32 val;
4697
4698         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
4699                 switch (ofs) {
4700                 case RCVLSC_MODE:
4701                 case DMAC_MODE:
4702                 case MBFREE_MODE:
4703                 case BUFMGR_MODE:
4704                 case MEMARB_MODE:
4705                         /* We can't enable/disable these bits of the
4706                          * 5705/5750, just say success.
4707                          */
4708                         return 0;
4709
4710                 default:
4711                         break;
4712                 };
4713         }
4714
4715         val = tr32(ofs);
4716         val &= ~enable_bit;
4717         tw32_f(ofs, val);
4718
4719         for (i = 0; i < MAX_WAIT_CNT; i++) {
4720                 udelay(100);
4721                 val = tr32(ofs);
4722                 if ((val & enable_bit) == 0)
4723                         break;
4724         }
4725
4726         if (i == MAX_WAIT_CNT && !silent) {
4727                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
4728                        "ofs=%lx enable_bit=%x\n",
4729                        ofs, enable_bit);
4730                 return -ENODEV;
4731         }
4732
4733         return 0;
4734 }
4735
4736 /* tp->lock is held. */
4737 static int tg3_abort_hw(struct tg3 *tp, int silent)
4738 {
4739         int i, err;
4740
4741         tg3_disable_ints(tp);
4742
4743         tp->rx_mode &= ~RX_MODE_ENABLE;
4744         tw32_f(MAC_RX_MODE, tp->rx_mode);
4745         udelay(10);
4746
4747         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
4748         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
4749         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
4750         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
4751         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
4752         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
4753
4754         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
4755         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
4756         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
4757         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
4758         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
4759         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
4760         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
4761
4762         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
4763         tw32_f(MAC_MODE, tp->mac_mode);
4764         udelay(40);
4765
4766         tp->tx_mode &= ~TX_MODE_ENABLE;
4767         tw32_f(MAC_TX_MODE, tp->tx_mode);
4768
4769         for (i = 0; i < MAX_WAIT_CNT; i++) {
4770                 udelay(100);
4771                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
4772                         break;
4773         }
4774         if (i >= MAX_WAIT_CNT) {
4775                 printk(KERN_ERR PFX "tg3_abort_hw timed out for %s, "
4776                        "TX_MODE_ENABLE will not clear MAC_TX_MODE=%08x\n",
4777                        tp->dev->name, tr32(MAC_TX_MODE));
4778                 err |= -ENODEV;
4779         }
4780
4781         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
4782         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
4783         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
4784
4785         tw32(FTQ_RESET, 0xffffffff);
4786         tw32(FTQ_RESET, 0x00000000);
4787
4788         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
4789         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
4790
4791         if (tp->hw_status)
4792                 memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4793         if (tp->hw_stats)
4794                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4795
4796         return err;
4797 }
4798
4799 /* tp->lock is held. */
4800 static int tg3_nvram_lock(struct tg3 *tp)
4801 {
4802         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4803                 int i;
4804
4805                 if (tp->nvram_lock_cnt == 0) {
4806                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
4807                         for (i = 0; i < 8000; i++) {
4808                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
4809                                         break;
4810                                 udelay(20);
4811                         }
4812                         if (i == 8000) {
4813                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
4814                                 return -ENODEV;
4815                         }
4816                 }
4817                 tp->nvram_lock_cnt++;
4818         }
4819         return 0;
4820 }
4821
4822 /* tp->lock is held. */
4823 static void tg3_nvram_unlock(struct tg3 *tp)
4824 {
4825         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4826                 if (tp->nvram_lock_cnt > 0)
4827                         tp->nvram_lock_cnt--;
4828                 if (tp->nvram_lock_cnt == 0)
4829                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
4830         }
4831 }
4832
4833 /* tp->lock is held. */
4834 static void tg3_enable_nvram_access(struct tg3 *tp)
4835 {
4836         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4837             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4838                 u32 nvaccess = tr32(NVRAM_ACCESS);
4839
4840                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
4841         }
4842 }
4843
4844 /* tp->lock is held. */
4845 static void tg3_disable_nvram_access(struct tg3 *tp)
4846 {
4847         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4848             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4849                 u32 nvaccess = tr32(NVRAM_ACCESS);
4850
4851                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
4852         }
4853 }
4854
4855 static void tg3_ape_send_event(struct tg3 *tp, u32 event)
4856 {
4857         int i;
4858         u32 apedata;
4859
4860         apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
4861         if (apedata != APE_SEG_SIG_MAGIC)
4862                 return;
4863
4864         apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
4865         if (apedata != APE_FW_STATUS_READY)
4866                 return;
4867
4868         /* Wait for up to 1 millisecond for APE to service previous event. */
4869         for (i = 0; i < 10; i++) {
4870                 if (tg3_ape_lock(tp, TG3_APE_LOCK_MEM))
4871                         return;
4872
4873                 apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
4874
4875                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
4876                         tg3_ape_write32(tp, TG3_APE_EVENT_STATUS,
4877                                         event | APE_EVENT_STATUS_EVENT_PENDING);
4878
4879                 tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
4880
4881                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
4882                         break;
4883
4884                 udelay(100);
4885         }
4886
4887         if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
4888                 tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
4889 }
4890
4891 static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
4892 {
4893         u32 event;
4894         u32 apedata;
4895
4896         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
4897                 return;
4898
4899         switch (kind) {
4900                 case RESET_KIND_INIT:
4901                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
4902                                         APE_HOST_SEG_SIG_MAGIC);
4903                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
4904                                         APE_HOST_SEG_LEN_MAGIC);
4905                         apedata = tg3_ape_read32(tp, TG3_APE_HOST_INIT_COUNT);
4906                         tg3_ape_write32(tp, TG3_APE_HOST_INIT_COUNT, ++apedata);
4907                         tg3_ape_write32(tp, TG3_APE_HOST_DRIVER_ID,
4908                                         APE_HOST_DRIVER_ID_MAGIC);
4909                         tg3_ape_write32(tp, TG3_APE_HOST_BEHAVIOR,
4910                                         APE_HOST_BEHAV_NO_PHYLOCK);
4911
4912                         event = APE_EVENT_STATUS_STATE_START;
4913                         break;
4914                 case RESET_KIND_SHUTDOWN:
4915                         event = APE_EVENT_STATUS_STATE_UNLOAD;
4916                         break;
4917                 case RESET_KIND_SUSPEND:
4918                         event = APE_EVENT_STATUS_STATE_SUSPEND;
4919                         break;
4920                 default:
4921                         return;
4922         }
4923
4924         event |= APE_EVENT_STATUS_DRIVER_EVNT | APE_EVENT_STATUS_STATE_CHNGE;
4925
4926         tg3_ape_send_event(tp, event);
4927 }
4928
4929 /* tp->lock is held. */
4930 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
4931 {
4932         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
4933                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
4934
4935         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4936                 switch (kind) {
4937                 case RESET_KIND_INIT:
4938                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4939                                       DRV_STATE_START);
4940                         break;
4941
4942                 case RESET_KIND_SHUTDOWN:
4943                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4944                                       DRV_STATE_UNLOAD);
4945                         break;
4946
4947                 case RESET_KIND_SUSPEND:
4948                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4949                                       DRV_STATE_SUSPEND);
4950                         break;
4951
4952                 default:
4953                         break;
4954                 };
4955         }
4956
4957         if (kind == RESET_KIND_INIT ||
4958             kind == RESET_KIND_SUSPEND)
4959                 tg3_ape_driver_state_change(tp, kind);
4960 }
4961
4962 /* tp->lock is held. */
4963 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
4964 {
4965         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4966                 switch (kind) {
4967                 case RESET_KIND_INIT:
4968                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4969                                       DRV_STATE_START_DONE);
4970                         break;
4971
4972                 case RESET_KIND_SHUTDOWN:
4973                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4974                                       DRV_STATE_UNLOAD_DONE);
4975                         break;
4976
4977                 default:
4978                         break;
4979                 };
4980         }
4981
4982         if (kind == RESET_KIND_SHUTDOWN)
4983                 tg3_ape_driver_state_change(tp, kind);
4984 }
4985
4986 /* tp->lock is held. */
4987 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
4988 {
4989         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4990                 switch (kind) {
4991                 case RESET_KIND_INIT:
4992                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4993                                       DRV_STATE_START);
4994                         break;
4995
4996                 case RESET_KIND_SHUTDOWN:
4997                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4998                                       DRV_STATE_UNLOAD);
4999                         break;
5000
5001                 case RESET_KIND_SUSPEND:
5002                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5003                                       DRV_STATE_SUSPEND);
5004                         break;
5005
5006                 default:
5007                         break;
5008                 };
5009         }
5010 }
5011
5012 static int tg3_poll_fw(struct tg3 *tp)
5013 {
5014         int i;
5015         u32 val;
5016
5017         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5018                 /* Wait up to 20ms for init done. */
5019                 for (i = 0; i < 200; i++) {
5020                         if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
5021                                 return 0;
5022                         udelay(100);
5023                 }
5024                 return -ENODEV;
5025         }
5026
5027         /* Wait for firmware initialization to complete. */
5028         for (i = 0; i < 100000; i++) {
5029                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
5030                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
5031                         break;
5032                 udelay(10);
5033         }
5034
5035         /* Chip might not be fitted with firmware.  Some Sun onboard
5036          * parts are configured like that.  So don't signal the timeout
5037          * of the above loop as an error, but do report the lack of
5038          * running firmware once.
5039          */
5040         if (i >= 100000 &&
5041             !(tp->tg3_flags2 & TG3_FLG2_NO_FWARE_REPORTED)) {
5042                 tp->tg3_flags2 |= TG3_FLG2_NO_FWARE_REPORTED;
5043
5044                 printk(KERN_INFO PFX "%s: No firmware running.\n",
5045                        tp->dev->name);
5046         }
5047
5048         return 0;
5049 }
5050
5051 /* Save PCI command register before chip reset */
5052 static void tg3_save_pci_state(struct tg3 *tp)
5053 {
5054         pci_read_config_word(tp->pdev, PCI_COMMAND, &tp->pci_cmd);
5055 }
5056
5057 /* Restore PCI state after chip reset */
5058 static void tg3_restore_pci_state(struct tg3 *tp)
5059 {
5060         u32 val;
5061
5062         /* Re-enable indirect register accesses. */
5063         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
5064                                tp->misc_host_ctrl);
5065
5066         /* Set MAX PCI retry to zero. */
5067         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
5068         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
5069             (tp->tg3_flags & TG3_FLAG_PCIX_MODE))
5070                 val |= PCISTATE_RETRY_SAME_DMA;
5071         /* Allow reads and writes to the APE register and memory space. */
5072         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
5073                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
5074                        PCISTATE_ALLOW_APE_SHMEM_WR;
5075         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
5076
5077         pci_write_config_word(tp->pdev, PCI_COMMAND, tp->pci_cmd);
5078
5079         if (!(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
5080                 pci_write_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
5081                                       tp->pci_cacheline_sz);
5082                 pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
5083                                       tp->pci_lat_timer);
5084         }
5085         /* Make sure PCI-X relaxed ordering bit is clear. */
5086         if (tp->pcix_cap) {
5087                 u16 pcix_cmd;
5088
5089                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
5090                                      &pcix_cmd);
5091                 pcix_cmd &= ~PCI_X_CMD_ERO;
5092                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
5093                                       pcix_cmd);
5094         }
5095
5096         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
5097
5098                 /* Chip reset on 5780 will reset MSI enable bit,
5099                  * so need to restore it.
5100                  */
5101                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
5102                         u16 ctrl;
5103
5104                         pci_read_config_word(tp->pdev,
5105                                              tp->msi_cap + PCI_MSI_FLAGS,
5106                                              &ctrl);
5107                         pci_write_config_word(tp->pdev,
5108                                               tp->msi_cap + PCI_MSI_FLAGS,
5109                                               ctrl | PCI_MSI_FLAGS_ENABLE);
5110                         val = tr32(MSGINT_MODE);
5111                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
5112                 }
5113         }
5114 }
5115
5116 static void tg3_stop_fw(struct tg3 *);
5117
5118 /* tp->lock is held. */
5119 static int tg3_chip_reset(struct tg3 *tp)
5120 {
5121         u32 val;
5122         void (*write_op)(struct tg3 *, u32, u32);
5123         int err;
5124
5125         tg3_nvram_lock(tp);
5126
5127         /* No matching tg3_nvram_unlock() after this because
5128          * chip reset below will undo the nvram lock.
5129          */
5130         tp->nvram_lock_cnt = 0;
5131
5132         /* GRC_MISC_CFG core clock reset will clear the memory
5133          * enable bit in PCI register 4 and the MSI enable bit
5134          * on some chips, so we save relevant registers here.
5135          */
5136         tg3_save_pci_state(tp);
5137
5138         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
5139             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
5140             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
5141             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
5142             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
5143                 tw32(GRC_FASTBOOT_PC, 0);
5144
5145         /*
5146          * We must avoid the readl() that normally takes place.
5147          * It locks machines, causes machine checks, and other
5148          * fun things.  So, temporarily disable the 5701
5149          * hardware workaround, while we do the reset.
5150          */
5151         write_op = tp->write32;
5152         if (write_op == tg3_write_flush_reg32)
5153                 tp->write32 = tg3_write32;
5154
5155         /* Prevent the irq handler from reading or writing PCI registers
5156          * during chip reset when the memory enable bit in the PCI command
5157          * register may be cleared.  The chip does not generate interrupt
5158          * at this time, but the irq handler may still be called due to irq
5159          * sharing or irqpoll.
5160          */
5161         tp->tg3_flags |= TG3_FLAG_CHIP_RESETTING;
5162         if (tp->hw_status) {
5163                 tp->hw_status->status = 0;
5164                 tp->hw_status->status_tag = 0;
5165         }
5166         tp->last_tag = 0;
5167         smp_mb();
5168         synchronize_irq(tp->pdev->irq);
5169
5170         /* do the reset */
5171         val = GRC_MISC_CFG_CORECLK_RESET;
5172
5173         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
5174                 if (tr32(0x7e2c) == 0x60) {
5175                         tw32(0x7e2c, 0x20);
5176                 }
5177                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
5178                         tw32(GRC_MISC_CFG, (1 << 29));
5179                         val |= (1 << 29);
5180                 }
5181         }
5182
5183         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5184                 tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
5185                 tw32(GRC_VCPU_EXT_CTRL,
5186                      tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
5187         }
5188
5189         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5190                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
5191         tw32(GRC_MISC_CFG, val);
5192
5193         /* restore 5701 hardware bug workaround write method */
5194         tp->write32 = write_op;
5195
5196         /* Unfortunately, we have to delay before the PCI read back.
5197          * Some 575X chips even will not respond to a PCI cfg access
5198          * when the reset command is given to the chip.
5199          *
5200          * How do these hardware designers expect things to work
5201          * properly if the PCI write is posted for a long period
5202          * of time?  It is always necessary to have some method by
5203          * which a register read back can occur to push the write
5204          * out which does the reset.
5205          *
5206          * For most tg3 variants the trick below was working.
5207          * Ho hum...
5208          */
5209         udelay(120);
5210
5211         /* Flush PCI posted writes.  The normal MMIO registers
5212          * are inaccessible at this time so this is the only
5213          * way to make this reliably (actually, this is no longer
5214          * the case, see above).  I tried to use indirect
5215          * register read/write but this upset some 5701 variants.
5216          */
5217         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
5218
5219         udelay(120);
5220
5221         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
5222                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
5223                         int i;
5224                         u32 cfg_val;
5225
5226                         /* Wait for link training to complete.  */
5227                         for (i = 0; i < 5000; i++)
5228                                 udelay(100);
5229
5230                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
5231                         pci_write_config_dword(tp->pdev, 0xc4,
5232                                                cfg_val | (1 << 15));
5233                 }
5234                 /* Set PCIE max payload size and clear error status.  */
5235                 pci_write_config_dword(tp->pdev, 0xd8, 0xf5000);
5236         }
5237
5238         tg3_restore_pci_state(tp);
5239
5240         tp->tg3_flags &= ~TG3_FLAG_CHIP_RESETTING;
5241
5242         val = 0;
5243         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
5244                 val = tr32(MEMARB_MODE);
5245         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
5246
5247         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
5248                 tg3_stop_fw(tp);
5249                 tw32(0x5000, 0x400);
5250         }
5251
5252         tw32(GRC_MODE, tp->grc_mode);
5253
5254         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
5255                 val = tr32(0xc4);
5256
5257                 tw32(0xc4, val | (1 << 15));
5258         }
5259
5260         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
5261             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
5262                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
5263                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
5264                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
5265                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
5266         }
5267
5268         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
5269                 tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
5270                 tw32_f(MAC_MODE, tp->mac_mode);
5271         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
5272                 tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
5273                 tw32_f(MAC_MODE, tp->mac_mode);
5274         } else
5275                 tw32_f(MAC_MODE, 0);
5276         udelay(40);
5277
5278         err = tg3_poll_fw(tp);
5279         if (err)
5280                 return err;
5281
5282         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
5283             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
5284                 val = tr32(0x7c00);
5285
5286                 tw32(0x7c00, val | (1 << 25));
5287         }
5288
5289         /* Reprobe ASF enable state.  */
5290         tp->tg3_flags &= ~TG3_FLAG_ENABLE_ASF;
5291         tp->tg3_flags2 &= ~TG3_FLG2_ASF_NEW_HANDSHAKE;
5292         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
5293         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
5294                 u32 nic_cfg;
5295
5296                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
5297                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
5298                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
5299                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
5300                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
5301                 }
5302         }
5303
5304         return 0;
5305 }
5306
5307 /* tp->lock is held. */
5308 static void tg3_stop_fw(struct tg3 *tp)
5309 {
5310         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) &&
5311            !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
5312                 u32 val;
5313                 int i;
5314
5315                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
5316                 val = tr32(GRC_RX_CPU_EVENT);
5317                 val |= (1 << 14);
5318                 tw32(GRC_RX_CPU_EVENT, val);
5319
5320                 /* Wait for RX cpu to ACK the event.  */
5321                 for (i = 0; i < 100; i++) {
5322                         if (!(tr32(GRC_RX_CPU_EVENT) & (1 << 14)))
5323                                 break;
5324                         udelay(1);
5325                 }
5326         }
5327 }
5328
5329 /* tp->lock is held. */
5330 static int tg3_halt(struct tg3 *tp, int kind, int silent)
5331 {
5332         int err;
5333
5334         tg3_stop_fw(tp);
5335
5336         tg3_write_sig_pre_reset(tp, kind);
5337
5338         tg3_abort_hw(tp, silent);
5339         err = tg3_chip_reset(tp);
5340
5341         tg3_write_sig_legacy(tp, kind);
5342         tg3_write_sig_post_reset(tp, kind);
5343
5344         if (err)
5345                 return err;
5346
5347         return 0;
5348 }
5349
5350 #define TG3_FW_RELEASE_MAJOR    0x0
5351 #define TG3_FW_RELASE_MINOR     0x0
5352 #define TG3_FW_RELEASE_FIX      0x0
5353 #define TG3_FW_START_ADDR       0x08000000
5354 #define TG3_FW_TEXT_ADDR        0x08000000
5355 #define TG3_FW_TEXT_LEN         0x9c0
5356 #define TG3_FW_RODATA_ADDR      0x080009c0
5357 #define TG3_FW_RODATA_LEN       0x60
5358 #define TG3_FW_DATA_ADDR        0x08000a40
5359 #define TG3_FW_DATA_LEN         0x20
5360 #define TG3_FW_SBSS_ADDR        0x08000a60
5361 #define TG3_FW_SBSS_LEN         0xc
5362 #define TG3_FW_BSS_ADDR         0x08000a70
5363 #define TG3_FW_BSS_LEN          0x10
5364
5365 static const u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
5366         0x00000000, 0x10000003, 0x00000000, 0x0000000d, 0x0000000d, 0x3c1d0800,
5367         0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100000, 0x0e000018, 0x00000000,
5368         0x0000000d, 0x3c1d0800, 0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100034,
5369         0x0e00021c, 0x00000000, 0x0000000d, 0x00000000, 0x00000000, 0x00000000,
5370         0x27bdffe0, 0x3c1cc000, 0xafbf0018, 0xaf80680c, 0x0e00004c, 0x241b2105,
5371         0x97850000, 0x97870002, 0x9782002c, 0x9783002e, 0x3c040800, 0x248409c0,
5372         0xafa00014, 0x00021400, 0x00621825, 0x00052c00, 0xafa30010, 0x8f860010,
5373         0x00e52825, 0x0e000060, 0x24070102, 0x3c02ac00, 0x34420100, 0x3c03ac01,
5374         0x34630100, 0xaf820490, 0x3c02ffff, 0xaf820494, 0xaf830498, 0xaf82049c,
5375         0x24020001, 0xaf825ce0, 0x0e00003f, 0xaf825d00, 0x0e000140, 0x00000000,
5376         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x2402ffff, 0xaf825404, 0x8f835400,
5377         0x34630400, 0xaf835400, 0xaf825404, 0x3c020800, 0x24420034, 0xaf82541c,
5378         0x03e00008, 0xaf805400, 0x00000000, 0x00000000, 0x3c020800, 0x34423000,
5379         0x3c030800, 0x34633000, 0x3c040800, 0x348437ff, 0x3c010800, 0xac220a64,
5380         0x24020040, 0x3c010800, 0xac220a68, 0x3c010800, 0xac200a60, 0xac600000,
5381         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
5382         0x00804821, 0x8faa0010, 0x3c020800, 0x8c420a60, 0x3c040800, 0x8c840a68,
5383         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010800, 0xac230a60, 0x14400003,
5384         0x00004021, 0x3c010800, 0xac200a60, 0x3c020800, 0x8c420a60, 0x3c030800,
5385         0x8c630a64, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
5386         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020800, 0x8c420a60,
5387         0x3c030800, 0x8c630a64, 0x8f84680c, 0x00021140, 0x00431021, 0xac440008,
5388         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
5389         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5390         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5391         0, 0, 0, 0, 0, 0,
5392         0x02000008, 0x00000000, 0x0a0001e3, 0x3c0a0001, 0x0a0001e3, 0x3c0a0002,
5393         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5394         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5395         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5396         0x0a0001e3, 0x3c0a0007, 0x0a0001e3, 0x3c0a0008, 0x0a0001e3, 0x3c0a0009,
5397         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000b,
5398         0x0a0001e3, 0x3c0a000c, 0x0a0001e3, 0x3c0a000d, 0x0a0001e3, 0x00000000,
5399         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000e, 0x0a0001e3, 0x00000000,
5400         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5401         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5402         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a0013, 0x0a0001e3, 0x3c0a0014,
5403         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5404         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5405         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5406         0x27bdffe0, 0x00001821, 0x00001021, 0xafbf0018, 0xafb10014, 0xafb00010,
5407         0x3c010800, 0x00220821, 0xac200a70, 0x3c010800, 0x00220821, 0xac200a74,
5408         0x3c010800, 0x00220821, 0xac200a78, 0x24630001, 0x1860fff5, 0x2442000c,
5409         0x24110001, 0x8f906810, 0x32020004, 0x14400005, 0x24040001, 0x3c020800,
5410         0x8c420a78, 0x18400003, 0x00002021, 0x0e000182, 0x00000000, 0x32020001,
5411         0x10400003, 0x00000000, 0x0e000169, 0x00000000, 0x0a000153, 0xaf915028,
5412         0x8fbf0018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020, 0x3c050800,
5413         0x8ca50a70, 0x3c060800, 0x8cc60a80, 0x3c070800, 0x8ce70a78, 0x27bdffe0,
5414         0x3c040800, 0x248409d0, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014,
5415         0x0e00017b, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x24020001,
5416         0x8f836810, 0x00821004, 0x00021027, 0x00621824, 0x03e00008, 0xaf836810,
5417         0x27bdffd8, 0xafbf0024, 0x1080002e, 0xafb00020, 0x8f825cec, 0xafa20018,
5418         0x8f825cec, 0x3c100800, 0x26100a78, 0xafa2001c, 0x34028000, 0xaf825cec,
5419         0x8e020000, 0x18400016, 0x00000000, 0x3c020800, 0x94420a74, 0x8fa3001c,
5420         0x000221c0, 0xac830004, 0x8fa2001c, 0x3c010800, 0x0e000201, 0xac220a74,
5421         0x10400005, 0x00000000, 0x8e020000, 0x24420001, 0x0a0001df, 0xae020000,
5422         0x3c020800, 0x8c420a70, 0x00021c02, 0x000321c0, 0x0a0001c5, 0xafa2001c,
5423         0x0e000201, 0x00000000, 0x1040001f, 0x00000000, 0x8e020000, 0x8fa3001c,
5424         0x24420001, 0x3c010800, 0xac230a70, 0x3c010800, 0xac230a74, 0x0a0001df,
5425         0xae020000, 0x3c100800, 0x26100a78, 0x8e020000, 0x18400028, 0x00000000,
5426         0x0e000201, 0x00000000, 0x14400024, 0x00000000, 0x8e020000, 0x3c030800,
5427         0x8c630a70, 0x2442ffff, 0xafa3001c, 0x18400006, 0xae020000, 0x00031402,
5428         0x000221c0, 0x8c820004, 0x3c010800, 0xac220a70, 0x97a2001e, 0x2442ff00,
5429         0x2c420300, 0x1440000b, 0x24024000, 0x3c040800, 0x248409dc, 0xafa00010,
5430         0xafa00014, 0x8fa6001c, 0x24050008, 0x0e000060, 0x00003821, 0x0a0001df,
5431         0x00000000, 0xaf825cf8, 0x3c020800, 0x8c420a40, 0x8fa3001c, 0x24420001,
5432         0xaf835cf8, 0x3c010800, 0xac220a40, 0x8fbf0024, 0x8fb00020, 0x03e00008,
5433         0x27bd0028, 0x27bdffe0, 0x3c040800, 0x248409e8, 0x00002821, 0x00003021,
5434         0x00003821, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x8fbf0018,
5435         0x03e00008, 0x27bd0020, 0x8f82680c, 0x8f85680c, 0x00021827, 0x0003182b,
5436         0x00031823, 0x00431024, 0x00441021, 0x00a2282b, 0x10a00006, 0x00000000,
5437         0x00401821, 0x8f82680c, 0x0043102b, 0x1440fffd, 0x00000000, 0x03e00008,
5438         0x00000000, 0x3c040800, 0x8c840000, 0x3c030800, 0x8c630a40, 0x0064102b,
5439         0x54400002, 0x00831023, 0x00641023, 0x2c420008, 0x03e00008, 0x38420001,
5440         0x27bdffe0, 0x00802821, 0x3c040800, 0x24840a00, 0x00003021, 0x00003821,
5441         0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x0a000216, 0x00000000,
5442         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000, 0x27bdffe0, 0x3c1cc000,
5443         0xafbf0018, 0x0e00004c, 0xaf80680c, 0x3c040800, 0x24840a10, 0x03802821,
5444         0x00003021, 0x00003821, 0xafa00010, 0x0e000060, 0xafa00014, 0x2402ffff,
5445         0xaf825404, 0x3c0200aa, 0x0e000234, 0xaf825434, 0x8fbf0018, 0x03e00008,
5446         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe8, 0xafb00010,
5447         0x24100001, 0xafbf0014, 0x3c01c003, 0xac200000, 0x8f826810, 0x30422000,
5448         0x10400003, 0x00000000, 0x0e000246, 0x00000000, 0x0a00023a, 0xaf905428,
5449         0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x27bdfff8, 0x8f845d0c,
5450         0x3c0200ff, 0x3c030800, 0x8c630a50, 0x3442fff8, 0x00821024, 0x1043001e,
5451         0x3c0500ff, 0x34a5fff8, 0x3c06c003, 0x3c074000, 0x00851824, 0x8c620010,
5452         0x3c010800, 0xac230a50, 0x30420008, 0x10400005, 0x00871025, 0x8cc20000,
5453         0x24420001, 0xacc20000, 0x00871025, 0xaf825d0c, 0x8fa20000, 0x24420001,
5454         0xafa20000, 0x8fa20000, 0x8fa20000, 0x24420001, 0xafa20000, 0x8fa20000,
5455         0x8f845d0c, 0x3c030800, 0x8c630a50, 0x00851024, 0x1443ffe8, 0x00851824,
5456         0x27bd0008, 0x03e00008, 0x00000000, 0x00000000, 0x00000000
5457 };
5458
5459 static const u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
5460         0x35373031, 0x726c7341, 0x00000000, 0x00000000, 0x53774576, 0x656e7430,
5461         0x00000000, 0x726c7045, 0x76656e74, 0x31000000, 0x556e6b6e, 0x45766e74,
5462         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
5463         0x00000000, 0x00000000, 0x4d61696e, 0x43707542, 0x00000000, 0x00000000,
5464         0x00000000
5465 };
5466
5467 #if 0 /* All zeros, don't eat up space with it. */
5468 u32 tg3FwData[(TG3_FW_DATA_LEN / sizeof(u32)) + 1] = {
5469         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5470         0x00000000, 0x00000000, 0x00000000, 0x00000000
5471 };
5472 #endif
5473
5474 #define RX_CPU_SCRATCH_BASE     0x30000
5475 #define RX_CPU_SCRATCH_SIZE     0x04000
5476 #define TX_CPU_SCRATCH_BASE     0x34000
5477 #define TX_CPU_SCRATCH_SIZE     0x04000
5478
5479 /* tp->lock is held. */
5480 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
5481 {
5482         int i;
5483
5484         BUG_ON(offset == TX_CPU_BASE &&
5485             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS));
5486
5487         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5488                 u32 val = tr32(GRC_VCPU_EXT_CTRL);
5489
5490                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
5491                 return 0;
5492         }
5493         if (offset == RX_CPU_BASE) {
5494                 for (i = 0; i < 10000; i++) {
5495                         tw32(offset + CPU_STATE, 0xffffffff);
5496                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5497                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5498                                 break;
5499                 }
5500
5501                 tw32(offset + CPU_STATE, 0xffffffff);
5502                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
5503                 udelay(10);
5504         } else {
5505                 for (i = 0; i < 10000; i++) {
5506                         tw32(offset + CPU_STATE, 0xffffffff);
5507                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5508                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5509                                 break;
5510                 }
5511         }
5512
5513         if (i >= 10000) {
5514                 printk(KERN_ERR PFX "tg3_reset_cpu timed out for %s, "
5515                        "and %s CPU\n",
5516                        tp->dev->name,
5517                        (offset == RX_CPU_BASE ? "RX" : "TX"));
5518                 return -ENODEV;
5519         }
5520
5521         /* Clear firmware's nvram arbitration. */
5522         if (tp->tg3_flags & TG3_FLAG_NVRAM)
5523                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
5524         return 0;
5525 }
5526
5527 struct fw_info {
5528         unsigned int text_base;
5529         unsigned int text_len;
5530         const u32 *text_data;
5531         unsigned int rodata_base;
5532         unsigned int rodata_len;
5533         const u32 *rodata_data;
5534         unsigned int data_base;
5535         unsigned int data_len;
5536         const u32 *data_data;
5537 };
5538
5539 /* tp->lock is held. */
5540 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
5541                                  int cpu_scratch_size, struct fw_info *info)
5542 {
5543         int err, lock_err, i;
5544         void (*write_op)(struct tg3 *, u32, u32);
5545
5546         if (cpu_base == TX_CPU_BASE &&
5547             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5548                 printk(KERN_ERR PFX "tg3_load_firmware_cpu: Trying to load "
5549                        "TX cpu firmware on %s which is 5705.\n",
5550                        tp->dev->name);
5551                 return -EINVAL;
5552         }
5553
5554         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5555                 write_op = tg3_write_mem;
5556         else
5557                 write_op = tg3_write_indirect_reg32;
5558
5559         /* It is possible that bootcode is still loading at this point.
5560          * Get the nvram lock first before halting the cpu.
5561          */
5562         lock_err = tg3_nvram_lock(tp);
5563         err = tg3_halt_cpu(tp, cpu_base);
5564         if (!lock_err)
5565                 tg3_nvram_unlock(tp);
5566         if (err)
5567                 goto out;
5568
5569         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
5570                 write_op(tp, cpu_scratch_base + i, 0);
5571         tw32(cpu_base + CPU_STATE, 0xffffffff);
5572         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
5573         for (i = 0; i < (info->text_len / sizeof(u32)); i++)
5574                 write_op(tp, (cpu_scratch_base +
5575                               (info->text_base & 0xffff) +
5576                               (i * sizeof(u32))),
5577                          (info->text_data ?
5578                           info->text_data[i] : 0));
5579         for (i = 0; i < (info->rodata_len / sizeof(u32)); i++)
5580                 write_op(tp, (cpu_scratch_base +
5581                               (info->rodata_base & 0xffff) +
5582                               (i * sizeof(u32))),
5583                          (info->rodata_data ?
5584                           info->rodata_data[i] : 0));
5585         for (i = 0; i < (info->data_len / sizeof(u32)); i++)
5586                 write_op(tp, (cpu_scratch_base +
5587                               (info->data_base & 0xffff) +
5588                               (i * sizeof(u32))),
5589                          (info->data_data ?
5590                           info->data_data[i] : 0));
5591
5592         err = 0;
5593
5594 out:
5595         return err;
5596 }
5597
5598 /* tp->lock is held. */
5599 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
5600 {
5601         struct fw_info info;
5602         int err, i;
5603
5604         info.text_base = TG3_FW_TEXT_ADDR;
5605         info.text_len = TG3_FW_TEXT_LEN;
5606         info.text_data = &tg3FwText[0];
5607         info.rodata_base = TG3_FW_RODATA_ADDR;
5608         info.rodata_len = TG3_FW_RODATA_LEN;
5609         info.rodata_data = &tg3FwRodata[0];
5610         info.data_base = TG3_FW_DATA_ADDR;
5611         info.data_len = TG3_FW_DATA_LEN;
5612         info.data_data = NULL;
5613
5614         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
5615                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
5616                                     &info);
5617         if (err)
5618                 return err;
5619
5620         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
5621                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
5622                                     &info);
5623         if (err)
5624                 return err;
5625
5626         /* Now startup only the RX cpu. */
5627         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5628         tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5629
5630         for (i = 0; i < 5; i++) {
5631                 if (tr32(RX_CPU_BASE + CPU_PC) == TG3_FW_TEXT_ADDR)
5632                         break;
5633                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5634                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
5635                 tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5636                 udelay(1000);
5637         }
5638         if (i >= 5) {
5639                 printk(KERN_ERR PFX "tg3_load_firmware fails for %s "
5640                        "to set RX CPU PC, is %08x should be %08x\n",
5641                        tp->dev->name, tr32(RX_CPU_BASE + CPU_PC),
5642                        TG3_FW_TEXT_ADDR);
5643                 return -ENODEV;
5644         }
5645         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5646         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
5647
5648         return 0;
5649 }
5650
5651
5652 #define TG3_TSO_FW_RELEASE_MAJOR        0x1
5653 #define TG3_TSO_FW_RELASE_MINOR         0x6
5654 #define TG3_TSO_FW_RELEASE_FIX          0x0
5655 #define TG3_TSO_FW_START_ADDR           0x08000000
5656 #define TG3_TSO_FW_TEXT_ADDR            0x08000000
5657 #define TG3_TSO_FW_TEXT_LEN             0x1aa0
5658 #define TG3_TSO_FW_RODATA_ADDR          0x08001aa0
5659 #define TG3_TSO_FW_RODATA_LEN           0x60
5660 #define TG3_TSO_FW_DATA_ADDR            0x08001b20
5661 #define TG3_TSO_FW_DATA_LEN             0x30
5662 #define TG3_TSO_FW_SBSS_ADDR            0x08001b50
5663 #define TG3_TSO_FW_SBSS_LEN             0x2c
5664 #define TG3_TSO_FW_BSS_ADDR             0x08001b80
5665 #define TG3_TSO_FW_BSS_LEN              0x894
5666
5667 static const u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
5668         0x0e000003, 0x00000000, 0x08001b24, 0x00000000, 0x10000003, 0x00000000,
5669         0x0000000d, 0x0000000d, 0x3c1d0800, 0x37bd4000, 0x03a0f021, 0x3c100800,
5670         0x26100000, 0x0e000010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5671         0xafbf0018, 0x0e0005d8, 0x34840002, 0x0e000668, 0x00000000, 0x3c030800,
5672         0x90631b68, 0x24020002, 0x3c040800, 0x24841aac, 0x14620003, 0x24050001,
5673         0x3c040800, 0x24841aa0, 0x24060006, 0x00003821, 0xafa00010, 0x0e00067c,
5674         0xafa00014, 0x8f625c50, 0x34420001, 0xaf625c50, 0x8f625c90, 0x34420001,
5675         0xaf625c90, 0x2402ffff, 0x0e000034, 0xaf625404, 0x8fbf0018, 0x03e00008,
5676         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c,
5677         0xafb20018, 0xafb10014, 0x0e00005b, 0xafb00010, 0x24120002, 0x24110001,
5678         0x8f706820, 0x32020100, 0x10400003, 0x00000000, 0x0e0000bb, 0x00000000,
5679         0x8f706820, 0x32022000, 0x10400004, 0x32020001, 0x0e0001f0, 0x24040001,
5680         0x32020001, 0x10400003, 0x00000000, 0x0e0000a3, 0x00000000, 0x3c020800,
5681         0x90421b98, 0x14520003, 0x00000000, 0x0e0004c0, 0x00000000, 0x0a00003c,
5682         0xaf715028, 0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008,
5683         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ac0, 0x00002821, 0x00003021,
5684         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x3c040800,
5685         0x248423d8, 0xa4800000, 0x3c010800, 0xa0201b98, 0x3c010800, 0xac201b9c,
5686         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5687         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bbc, 0x8f624434, 0x3c010800,
5688         0xac221b88, 0x8f624438, 0x3c010800, 0xac221b8c, 0x8f624410, 0xac80f7a8,
5689         0x3c010800, 0xac201b84, 0x3c010800, 0xac2023e0, 0x3c010800, 0xac2023c8,
5690         0x3c010800, 0xac2023cc, 0x3c010800, 0xac202400, 0x3c010800, 0xac221b90,
5691         0x8f620068, 0x24030007, 0x00021702, 0x10430005, 0x00000000, 0x8f620068,
5692         0x00021702, 0x14400004, 0x24020001, 0x3c010800, 0x0a000097, 0xac20240c,
5693         0xac820034, 0x3c040800, 0x24841acc, 0x3c050800, 0x8ca5240c, 0x00003021,
5694         0x00003821, 0xafa00010, 0x0e00067c, 0xafa00014, 0x8fbf0018, 0x03e00008,
5695         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ad8, 0x00002821, 0x00003021,
5696         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x0e00005b,
5697         0x00000000, 0x0e0000b4, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5698         0x24020001, 0x8f636820, 0x00821004, 0x00021027, 0x00621824, 0x03e00008,
5699         0xaf636820, 0x27bdffd0, 0xafbf002c, 0xafb60028, 0xafb50024, 0xafb40020,
5700         0xafb3001c, 0xafb20018, 0xafb10014, 0xafb00010, 0x8f675c5c, 0x3c030800,
5701         0x24631bbc, 0x8c620000, 0x14470005, 0x3c0200ff, 0x3c020800, 0x90421b98,
5702         0x14400119, 0x3c0200ff, 0x3442fff8, 0x00e28824, 0xac670000, 0x00111902,
5703         0x306300ff, 0x30e20003, 0x000211c0, 0x00622825, 0x00a04021, 0x00071602,
5704         0x3c030800, 0x90631b98, 0x3044000f, 0x14600036, 0x00804821, 0x24020001,
5705         0x3c010800, 0xa0221b98, 0x00051100, 0x00821025, 0x3c010800, 0xac201b9c,
5706         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5707         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bb0, 0x3c010800, 0xac201bb4,
5708         0x3c010800, 0xa42223d8, 0x9622000c, 0x30437fff, 0x3c010800, 0xa4222410,
5709         0x30428000, 0x3c010800, 0xa4231bc6, 0x10400005, 0x24020001, 0x3c010800,
5710         0xac2223f4, 0x0a000102, 0x2406003e, 0x24060036, 0x3c010800, 0xac2023f4,
5711         0x9622000a, 0x3c030800, 0x94631bc6, 0x3c010800, 0xac2023f0, 0x3c010800,
5712         0xac2023f8, 0x00021302, 0x00021080, 0x00c21021, 0x00621821, 0x3c010800,
5713         0xa42223d0, 0x3c010800, 0x0a000115, 0xa4231b96, 0x9622000c, 0x3c010800,
5714         0xa42223ec, 0x3c040800, 0x24841b9c, 0x8c820000, 0x00021100, 0x3c010800,
5715         0x00220821, 0xac311bc8, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5716         0xac271bcc, 0x8c820000, 0x25030001, 0x306601ff, 0x00021100, 0x3c010800,
5717         0x00220821, 0xac261bd0, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5718         0xac291bd4, 0x96230008, 0x3c020800, 0x8c421bac, 0x00432821, 0x3c010800,
5719         0xac251bac, 0x9622000a, 0x30420004, 0x14400018, 0x00061100, 0x8f630c14,
5720         0x3063000f, 0x2c620002, 0x1440000b, 0x3c02c000, 0x8f630c14, 0x3c020800,
5721         0x8c421b40, 0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002,
5722         0x1040fff7, 0x3c02c000, 0x00e21825, 0xaf635c5c, 0x8f625c50, 0x30420002,
5723         0x10400014, 0x00000000, 0x0a000147, 0x00000000, 0x3c030800, 0x8c631b80,
5724         0x3c040800, 0x94841b94, 0x01221025, 0x3c010800, 0xa42223da, 0x24020001,
5725         0x3c010800, 0xac221bb8, 0x24630001, 0x0085202a, 0x3c010800, 0x10800003,
5726         0xac231b80, 0x3c010800, 0xa4251b94, 0x3c060800, 0x24c61b9c, 0x8cc20000,
5727         0x24420001, 0xacc20000, 0x28420080, 0x14400005, 0x00000000, 0x0e000656,
5728         0x24040002, 0x0a0001e6, 0x00000000, 0x3c020800, 0x8c421bb8, 0x10400078,
5729         0x24020001, 0x3c050800, 0x90a51b98, 0x14a20072, 0x00000000, 0x3c150800,
5730         0x96b51b96, 0x3c040800, 0x8c841bac, 0x32a3ffff, 0x0083102a, 0x1440006c,
5731         0x00000000, 0x14830003, 0x00000000, 0x3c010800, 0xac2523f0, 0x1060005c,
5732         0x00009021, 0x24d60004, 0x0060a021, 0x24d30014, 0x8ec20000, 0x00028100,
5733         0x3c110800, 0x02308821, 0x0e000625, 0x8e311bc8, 0x00402821, 0x10a00054,
5734         0x00000000, 0x9628000a, 0x31020040, 0x10400005, 0x2407180c, 0x8e22000c,
5735         0x2407188c, 0x00021400, 0xaca20018, 0x3c030800, 0x00701821, 0x8c631bd0,
5736         0x3c020800, 0x00501021, 0x8c421bd4, 0x00031d00, 0x00021400, 0x00621825,
5737         0xaca30014, 0x8ec30004, 0x96220008, 0x00432023, 0x3242ffff, 0x3083ffff,
5738         0x00431021, 0x0282102a, 0x14400002, 0x02b23023, 0x00803021, 0x8e620000,
5739         0x30c4ffff, 0x00441021, 0xae620000, 0x8e220000, 0xaca20000, 0x8e220004,
5740         0x8e63fff4, 0x00431021, 0xaca20004, 0xa4a6000e, 0x8e62fff4, 0x00441021,
5741         0xae62fff4, 0x96230008, 0x0043102a, 0x14400005, 0x02469021, 0x8e62fff0,
5742         0xae60fff4, 0x24420001, 0xae62fff0, 0xaca00008, 0x3242ffff, 0x14540008,
5743         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x24020905, 0xa4a2000c,
5744         0x0a0001cb, 0x34e70020, 0xa4a2000c, 0x3c020800, 0x8c4223f0, 0x10400003,
5745         0x3c024b65, 0x0a0001d3, 0x34427654, 0x3c02b49a, 0x344289ab, 0xaca2001c,
5746         0x30e2ffff, 0xaca20010, 0x0e0005a2, 0x00a02021, 0x3242ffff, 0x0054102b,
5747         0x1440ffa9, 0x00000000, 0x24020002, 0x3c010800, 0x0a0001e6, 0xa0221b98,
5748         0x8ec2083c, 0x24420001, 0x0a0001e6, 0xaec2083c, 0x0e0004c0, 0x00000000,
5749         0x8fbf002c, 0x8fb60028, 0x8fb50024, 0x8fb40020, 0x8fb3001c, 0x8fb20018,
5750         0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0030, 0x27bdffd0, 0xafbf0028,
5751         0xafb30024, 0xafb20020, 0xafb1001c, 0xafb00018, 0x8f725c9c, 0x3c0200ff,
5752         0x3442fff8, 0x3c070800, 0x24e71bb4, 0x02428824, 0x9623000e, 0x8ce20000,
5753         0x00431021, 0xace20000, 0x8e220010, 0x30420020, 0x14400011, 0x00809821,
5754         0x0e00063b, 0x02202021, 0x3c02c000, 0x02421825, 0xaf635c9c, 0x8f625c90,
5755         0x30420002, 0x1040011e, 0x00000000, 0xaf635c9c, 0x8f625c90, 0x30420002,
5756         0x10400119, 0x00000000, 0x0a00020d, 0x00000000, 0x8e240008, 0x8e230014,
5757         0x00041402, 0x000231c0, 0x00031502, 0x304201ff, 0x2442ffff, 0x3042007f,
5758         0x00031942, 0x30637800, 0x00021100, 0x24424000, 0x00624821, 0x9522000a,
5759         0x3084ffff, 0x30420008, 0x104000b0, 0x000429c0, 0x3c020800, 0x8c422400,
5760         0x14400024, 0x24c50008, 0x94c20014, 0x3c010800, 0xa42223d0, 0x8cc40010,
5761         0x00041402, 0x3c010800, 0xa42223d2, 0x3c010800, 0xa42423d4, 0x94c2000e,
5762         0x3083ffff, 0x00431023, 0x3c010800, 0xac222408, 0x94c2001a, 0x3c010800,
5763         0xac262400, 0x3c010800, 0xac322404, 0x3c010800, 0xac2223fc, 0x3c02c000,
5764         0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e5, 0x00000000,
5765         0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e0, 0x00000000, 0x0a000246,
5766         0x00000000, 0x94c2000e, 0x3c030800, 0x946323d4, 0x00434023, 0x3103ffff,
5767         0x2c620008, 0x1040001c, 0x00000000, 0x94c20014, 0x24420028, 0x00a22821,
5768         0x00031042, 0x1840000b, 0x00002021, 0x24e60848, 0x00403821, 0x94a30000,
5769         0x8cc20000, 0x24840001, 0x00431021, 0xacc20000, 0x0087102a, 0x1440fff9,
5770         0x24a50002, 0x31020001, 0x1040001f, 0x3c024000, 0x3c040800, 0x248423fc,
5771         0xa0a00001, 0x94a30000, 0x8c820000, 0x00431021, 0x0a000285, 0xac820000,
5772         0x8f626800, 0x3c030010, 0x00431024, 0x10400009, 0x00000000, 0x94c2001a,
5773         0x3c030800, 0x8c6323fc, 0x00431021, 0x3c010800, 0xac2223fc, 0x0a000286,
5774         0x3c024000, 0x94c2001a, 0x94c4001c, 0x3c030800, 0x8c6323fc, 0x00441023,
5775         0x00621821, 0x3c010800, 0xac2323fc, 0x3c024000, 0x02421825, 0xaf635c9c,
5776         0x8f625c90, 0x30420002, 0x1440fffc, 0x00000000, 0x9522000a, 0x30420010,
5777         0x1040009b, 0x00000000, 0x3c030800, 0x946323d4, 0x3c070800, 0x24e72400,
5778         0x8ce40000, 0x8f626800, 0x24630030, 0x00832821, 0x3c030010, 0x00431024,
5779         0x1440000a, 0x00000000, 0x94a20004, 0x3c040800, 0x8c842408, 0x3c030800,
5780         0x8c6323fc, 0x00441023, 0x00621821, 0x3c010800, 0xac2323fc, 0x3c040800,
5781         0x8c8423fc, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402, 0x00822021,
5782         0x00041027, 0xa4a20006, 0x3c030800, 0x8c632404, 0x3c0200ff, 0x3442fff8,
5783         0x00628824, 0x96220008, 0x24050001, 0x24034000, 0x000231c0, 0x00801021,
5784         0xa4c2001a, 0xa4c0001c, 0xace00000, 0x3c010800, 0xac251b60, 0xaf635cb8,
5785         0x8f625cb0, 0x30420002, 0x10400003, 0x00000000, 0x3c010800, 0xac201b60,
5786         0x8e220008, 0xaf625cb8, 0x8f625cb0, 0x30420002, 0x10400003, 0x00000000,
5787         0x3c010800, 0xac201b60, 0x3c020800, 0x8c421b60, 0x1040ffec, 0x00000000,
5788         0x3c040800, 0x0e00063b, 0x8c842404, 0x0a00032a, 0x00000000, 0x3c030800,
5789         0x90631b98, 0x24020002, 0x14620003, 0x3c034b65, 0x0a0002e1, 0x00008021,
5790         0x8e22001c, 0x34637654, 0x10430002, 0x24100002, 0x24100001, 0x00c02021,
5791         0x0e000350, 0x02003021, 0x24020003, 0x3c010800, 0xa0221b98, 0x24020002,
5792         0x1202000a, 0x24020001, 0x3c030800, 0x8c6323f0, 0x10620006, 0x00000000,
5793         0x3c020800, 0x944223d8, 0x00021400, 0x0a00031f, 0xae220014, 0x3c040800,
5794         0x248423da, 0x94820000, 0x00021400, 0xae220014, 0x3c020800, 0x8c421bbc,
5795         0x3c03c000, 0x3c010800, 0xa0201b98, 0x00431025, 0xaf625c5c, 0x8f625c50,
5796         0x30420002, 0x10400009, 0x00000000, 0x2484f7e2, 0x8c820000, 0x00431025,
5797         0xaf625c5c, 0x8f625c50, 0x30420002, 0x1440fffa, 0x00000000, 0x3c020800,
5798         0x24421b84, 0x8c430000, 0x24630001, 0xac430000, 0x8f630c14, 0x3063000f,
5799         0x2c620002, 0x1440000c, 0x3c024000, 0x8f630c14, 0x3c020800, 0x8c421b40,
5800         0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7,
5801         0x00000000, 0x3c024000, 0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002,
5802         0x1440fffc, 0x00000000, 0x12600003, 0x00000000, 0x0e0004c0, 0x00000000,
5803         0x8fbf0028, 0x8fb30024, 0x8fb20020, 0x8fb1001c, 0x8fb00018, 0x03e00008,
5804         0x27bd0030, 0x8f634450, 0x3c040800, 0x24841b88, 0x8c820000, 0x00031c02,
5805         0x0043102b, 0x14400007, 0x3c038000, 0x8c840004, 0x8f624450, 0x00021c02,
5806         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5807         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3c024000,
5808         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00000000,
5809         0x03e00008, 0x00000000, 0x27bdffe0, 0x00805821, 0x14c00011, 0x256e0008,
5810         0x3c020800, 0x8c4223f4, 0x10400007, 0x24020016, 0x3c010800, 0xa42223d2,
5811         0x2402002a, 0x3c010800, 0x0a000364, 0xa42223d4, 0x8d670010, 0x00071402,
5812         0x3c010800, 0xa42223d2, 0x3c010800, 0xa42723d4, 0x3c040800, 0x948423d4,
5813         0x3c030800, 0x946323d2, 0x95cf0006, 0x3c020800, 0x944223d0, 0x00832023,
5814         0x01e2c023, 0x3065ffff, 0x24a20028, 0x01c24821, 0x3082ffff, 0x14c0001a,
5815         0x01226021, 0x9582000c, 0x3042003f, 0x3c010800, 0xa42223d6, 0x95820004,
5816         0x95830006, 0x3c010800, 0xac2023e4, 0x3c010800, 0xac2023e8, 0x00021400,
5817         0x00431025, 0x3c010800, 0xac221bc0, 0x95220004, 0x3c010800, 0xa4221bc4,
5818         0x95230002, 0x01e51023, 0x0043102a, 0x10400010, 0x24020001, 0x3c010800,
5819         0x0a000398, 0xac2223f8, 0x3c030800, 0x8c6323e8, 0x3c020800, 0x94421bc4,
5820         0x00431021, 0xa5220004, 0x3c020800, 0x94421bc0, 0xa5820004, 0x3c020800,
5821         0x8c421bc0, 0xa5820006, 0x3c020800, 0x8c4223f0, 0x3c0d0800, 0x8dad23e4,
5822         0x3c0a0800, 0x144000e5, 0x8d4a23e8, 0x3c020800, 0x94421bc4, 0x004a1821,
5823         0x3063ffff, 0x0062182b, 0x24020002, 0x10c2000d, 0x01435023, 0x3c020800,
5824         0x944223d6, 0x30420009, 0x10400008, 0x00000000, 0x9582000c, 0x3042fff6,
5825         0xa582000c, 0x3c020800, 0x944223d6, 0x30420009, 0x01a26823, 0x3c020800,
5826         0x8c4223f8, 0x1040004a, 0x01203821, 0x3c020800, 0x944223d2, 0x00004021,
5827         0xa520000a, 0x01e21023, 0xa5220002, 0x3082ffff, 0x00021042, 0x18400008,
5828         0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021, 0x0103102a,
5829         0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061402,
5830         0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021, 0x2527000c,
5831         0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004, 0x1440fffb,
5832         0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023, 0x01803821,
5833         0x3082ffff, 0xa4e00010, 0x00621821, 0x00021042, 0x18400010, 0x00c33021,
5834         0x00404821, 0x94e20000, 0x24e70002, 0x00c23021, 0x30e2007f, 0x14400006,
5835         0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80, 0x00625824, 0x25670008,
5836         0x0109102a, 0x1440fff3, 0x00000000, 0x30820001, 0x10400005, 0x00061c02,
5837         0xa0e00001, 0x94e20000, 0x00c23021, 0x00061c02, 0x30c2ffff, 0x00623021,
5838         0x00061402, 0x00c23021, 0x0a00047d, 0x30c6ffff, 0x24020002, 0x14c20081,
5839         0x00000000, 0x3c020800, 0x8c42240c, 0x14400007, 0x00000000, 0x3c020800,
5840         0x944223d2, 0x95230002, 0x01e21023, 0x10620077, 0x00000000, 0x3c020800,
5841         0x944223d2, 0x01e21023, 0xa5220002, 0x3c020800, 0x8c42240c, 0x1040001a,
5842         0x31e3ffff, 0x8dc70010, 0x3c020800, 0x94421b96, 0x00e04021, 0x00072c02,
5843         0x00aa2021, 0x00431023, 0x00823823, 0x00072402, 0x30e2ffff, 0x00823821,
5844         0x00071027, 0xa522000a, 0x3102ffff, 0x3c040800, 0x948423d4, 0x00453023,
5845         0x00e02821, 0x00641823, 0x006d1821, 0x00c33021, 0x00061c02, 0x30c2ffff,
5846         0x0a00047d, 0x00623021, 0x01203821, 0x00004021, 0x3082ffff, 0x00021042,
5847         0x18400008, 0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021,
5848         0x0103102a, 0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021,
5849         0x00061402, 0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021,
5850         0x2527000c, 0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004,
5851         0x1440fffb, 0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023,
5852         0x01803821, 0x3082ffff, 0xa4e00010, 0x3c040800, 0x948423d4, 0x00621821,
5853         0x00c33021, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061c02, 0x3c020800,
5854         0x944223d0, 0x00c34821, 0x00441023, 0x00021fc2, 0x00431021, 0x00021043,
5855         0x18400010, 0x00003021, 0x00402021, 0x94e20000, 0x24e70002, 0x00c23021,
5856         0x30e2007f, 0x14400006, 0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80,
5857         0x00625824, 0x25670008, 0x0104102a, 0x1440fff3, 0x00000000, 0x3c020800,
5858         0x944223ec, 0x00c23021, 0x3122ffff, 0x00c23021, 0x00061c02, 0x30c2ffff,
5859         0x00623021, 0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010,
5860         0xadc00014, 0x0a00049d, 0xadc00000, 0x8dc70010, 0x00e04021, 0x11400007,
5861         0x00072c02, 0x00aa3021, 0x00061402, 0x30c3ffff, 0x00433021, 0x00061402,
5862         0x00c22821, 0x00051027, 0xa522000a, 0x3c030800, 0x946323d4, 0x3102ffff,
5863         0x01e21021, 0x00433023, 0x00cd3021, 0x00061c02, 0x30c2ffff, 0x00623021,
5864         0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010, 0x3102ffff,
5865         0x00051c00, 0x00431025, 0xadc20010, 0x3c020800, 0x8c4223f4, 0x10400005,
5866         0x2de205eb, 0x14400002, 0x25e2fff2, 0x34028870, 0xa5c20034, 0x3c030800,
5867         0x246323e8, 0x8c620000, 0x24420001, 0xac620000, 0x3c040800, 0x8c8423e4,
5868         0x3c020800, 0x8c421bc0, 0x3303ffff, 0x00832021, 0x00431821, 0x0062102b,
5869         0x3c010800, 0xac2423e4, 0x10400003, 0x2482ffff, 0x3c010800, 0xac2223e4,
5870         0x3c010800, 0xac231bc0, 0x03e00008, 0x27bd0020, 0x27bdffb8, 0x3c050800,
5871         0x24a51b96, 0xafbf0044, 0xafbe0040, 0xafb7003c, 0xafb60038, 0xafb50034,
5872         0xafb40030, 0xafb3002c, 0xafb20028, 0xafb10024, 0xafb00020, 0x94a90000,
5873         0x3c020800, 0x944223d0, 0x3c030800, 0x8c631bb0, 0x3c040800, 0x8c841bac,
5874         0x01221023, 0x0064182a, 0xa7a9001e, 0x106000be, 0xa7a20016, 0x24be0022,
5875         0x97b6001e, 0x24b3001a, 0x24b70016, 0x8fc20000, 0x14400008, 0x00000000,
5876         0x8fc2fff8, 0x97a30016, 0x8fc4fff4, 0x00431021, 0x0082202a, 0x148000b0,
5877         0x00000000, 0x97d50818, 0x32a2ffff, 0x104000a3, 0x00009021, 0x0040a021,
5878         0x00008821, 0x0e000625, 0x00000000, 0x00403021, 0x14c00007, 0x00000000,
5879         0x3c020800, 0x8c4223dc, 0x24420001, 0x3c010800, 0x0a000596, 0xac2223dc,
5880         0x3c100800, 0x02118021, 0x8e101bc8, 0x9608000a, 0x31020040, 0x10400005,
5881         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x31020080,
5882         0x54400001, 0x34e70010, 0x3c020800, 0x00511021, 0x8c421bd0, 0x3c030800,
5883         0x00711821, 0x8c631bd4, 0x00021500, 0x00031c00, 0x00431025, 0xacc20014,
5884         0x96040008, 0x3242ffff, 0x00821021, 0x0282102a, 0x14400002, 0x02b22823,
5885         0x00802821, 0x8e020000, 0x02459021, 0xacc20000, 0x8e020004, 0x00c02021,
5886         0x26310010, 0xac820004, 0x30e2ffff, 0xac800008, 0xa485000e, 0xac820010,
5887         0x24020305, 0x0e0005a2, 0xa482000c, 0x3242ffff, 0x0054102b, 0x1440ffc5,
5888         0x3242ffff, 0x0a00058e, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5889         0x10400067, 0x00000000, 0x8e62fff0, 0x00028900, 0x3c100800, 0x02118021,
5890         0x0e000625, 0x8e101bc8, 0x00403021, 0x14c00005, 0x00000000, 0x8e62082c,
5891         0x24420001, 0x0a000596, 0xae62082c, 0x9608000a, 0x31020040, 0x10400005,
5892         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x3c020800,
5893         0x00511021, 0x8c421bd0, 0x3c030800, 0x00711821, 0x8c631bd4, 0x00021500,
5894         0x00031c00, 0x00431025, 0xacc20014, 0x8e63fff4, 0x96020008, 0x00432023,
5895         0x3242ffff, 0x3083ffff, 0x00431021, 0x02c2102a, 0x10400003, 0x00802821,
5896         0x97a9001e, 0x01322823, 0x8e620000, 0x30a4ffff, 0x00441021, 0xae620000,
5897         0xa4c5000e, 0x8e020000, 0xacc20000, 0x8e020004, 0x8e63fff4, 0x00431021,
5898         0xacc20004, 0x8e63fff4, 0x96020008, 0x00641821, 0x0062102a, 0x14400006,
5899         0x02459021, 0x8e62fff0, 0xae60fff4, 0x24420001, 0x0a000571, 0xae62fff0,
5900         0xae63fff4, 0xacc00008, 0x3242ffff, 0x10560003, 0x31020004, 0x10400006,
5901         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x34e70020, 0x24020905,
5902         0xa4c2000c, 0x8ee30000, 0x8ee20004, 0x14620007, 0x3c02b49a, 0x8ee20860,
5903         0x54400001, 0x34e70400, 0x3c024b65, 0x0a000588, 0x34427654, 0x344289ab,
5904         0xacc2001c, 0x30e2ffff, 0xacc20010, 0x0e0005a2, 0x00c02021, 0x3242ffff,
5905         0x0056102b, 0x1440ff9b, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5906         0x1440ff48, 0x00000000, 0x8fbf0044, 0x8fbe0040, 0x8fb7003c, 0x8fb60038,
5907         0x8fb50034, 0x8fb40030, 0x8fb3002c, 0x8fb20028, 0x8fb10024, 0x8fb00020,
5908         0x03e00008, 0x27bd0048, 0x27bdffe8, 0xafbf0014, 0xafb00010, 0x8f624450,
5909         0x8f634410, 0x0a0005b1, 0x00808021, 0x8f626820, 0x30422000, 0x10400003,
5910         0x00000000, 0x0e0001f0, 0x00002021, 0x8f624450, 0x8f634410, 0x3042ffff,
5911         0x0043102b, 0x1440fff5, 0x00000000, 0x8f630c14, 0x3063000f, 0x2c620002,
5912         0x1440000b, 0x00000000, 0x8f630c14, 0x3c020800, 0x8c421b40, 0x3063000f,
5913         0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7, 0x00000000,
5914         0xaf705c18, 0x8f625c10, 0x30420002, 0x10400009, 0x00000000, 0x8f626820,
5915         0x30422000, 0x1040fff8, 0x00000000, 0x0e0001f0, 0x00002021, 0x0a0005c4,
5916         0x00000000, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000,
5917         0x00000000, 0x00000000, 0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010,
5918         0xaf60680c, 0x8f626804, 0x34420082, 0xaf626804, 0x8f634000, 0x24020b50,
5919         0x3c010800, 0xac221b54, 0x24020b78, 0x3c010800, 0xac221b64, 0x34630002,
5920         0xaf634000, 0x0e000605, 0x00808021, 0x3c010800, 0xa0221b68, 0x304200ff,
5921         0x24030002, 0x14430005, 0x00000000, 0x3c020800, 0x8c421b54, 0x0a0005f8,
5922         0xac5000c0, 0x3c020800, 0x8c421b54, 0xac5000bc, 0x8f624434, 0x8f634438,
5923         0x8f644410, 0x3c010800, 0xac221b5c, 0x3c010800, 0xac231b6c, 0x3c010800,
5924         0xac241b58, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x3c040800,
5925         0x8c870000, 0x3c03aa55, 0x3463aa55, 0x3c06c003, 0xac830000, 0x8cc20000,
5926         0x14430007, 0x24050002, 0x3c0355aa, 0x346355aa, 0xac830000, 0x8cc20000,
5927         0x50430001, 0x24050001, 0x3c020800, 0xac470000, 0x03e00008, 0x00a01021,
5928         0x27bdfff8, 0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe,
5929         0x00000000, 0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008,
5930         0x27bd0008, 0x8f634450, 0x3c020800, 0x8c421b5c, 0x00031c02, 0x0043102b,
5931         0x14400008, 0x3c038000, 0x3c040800, 0x8c841b6c, 0x8f624450, 0x00021c02,
5932         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5933         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff,
5934         0x2442e000, 0x2c422001, 0x14400003, 0x3c024000, 0x0a000648, 0x2402ffff,
5935         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021,
5936         0x03e00008, 0x00000000, 0x8f624450, 0x3c030800, 0x8c631b58, 0x0a000651,
5937         0x3042ffff, 0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000,
5938         0x03e00008, 0x00000000, 0x27bdffe0, 0x00802821, 0x3c040800, 0x24841af0,
5939         0x00003021, 0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014,
5940         0x0a000660, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000,
5941         0x00000000, 0x00000000, 0x3c020800, 0x34423000, 0x3c030800, 0x34633000,
5942         0x3c040800, 0x348437ff, 0x3c010800, 0xac221b74, 0x24020040, 0x3c010800,
5943         0xac221b78, 0x3c010800, 0xac201b70, 0xac600000, 0x24630004, 0x0083102b,
5944         0x5040fffd, 0xac600000, 0x03e00008, 0x00000000, 0x00804821, 0x8faa0010,
5945         0x3c020800, 0x8c421b70, 0x3c040800, 0x8c841b78, 0x8fab0014, 0x24430001,
5946         0x0044102b, 0x3c010800, 0xac231b70, 0x14400003, 0x00004021, 0x3c010800,
5947         0xac201b70, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74, 0x91240000,
5948         0x00021140, 0x00431021, 0x00481021, 0x25080001, 0xa0440000, 0x29020008,
5949         0x1440fff4, 0x25290001, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74,
5950         0x8f64680c, 0x00021140, 0x00431021, 0xac440008, 0xac45000c, 0xac460010,
5951         0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c, 0x00000000, 0x00000000,
5952 };
5953
5954 static const u32 tg3TsoFwRodata[] = {
5955         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
5956         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x496e0000, 0x73746b6f,
5957         0x66662a2a, 0x00000000, 0x53774576, 0x656e7430, 0x00000000, 0x00000000,
5958         0x00000000, 0x00000000, 0x66617461, 0x6c457272, 0x00000000, 0x00000000,
5959         0x00000000,
5960 };
5961
5962 static const u32 tg3TsoFwData[] = {
5963         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x362e3000, 0x00000000,
5964         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5965         0x00000000,
5966 };
5967
5968 /* 5705 needs a special version of the TSO firmware.  */
5969 #define TG3_TSO5_FW_RELEASE_MAJOR       0x1
5970 #define TG3_TSO5_FW_RELASE_MINOR        0x2
5971 #define TG3_TSO5_FW_RELEASE_FIX         0x0
5972 #define TG3_TSO5_FW_START_ADDR          0x00010000
5973 #define TG3_TSO5_FW_TEXT_ADDR           0x00010000
5974 #define TG3_TSO5_FW_TEXT_LEN            0xe90
5975 #define TG3_TSO5_FW_RODATA_ADDR         0x00010e90
5976 #define TG3_TSO5_FW_RODATA_LEN          0x50
5977 #define TG3_TSO5_FW_DATA_ADDR           0x00010f00
5978 #define TG3_TSO5_FW_DATA_LEN            0x20
5979 #define TG3_TSO5_FW_SBSS_ADDR           0x00010f20
5980 #define TG3_TSO5_FW_SBSS_LEN            0x28
5981 #define TG3_TSO5_FW_BSS_ADDR            0x00010f50
5982 #define TG3_TSO5_FW_BSS_LEN             0x88
5983
5984 static const u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
5985         0x0c004003, 0x00000000, 0x00010f04, 0x00000000, 0x10000003, 0x00000000,
5986         0x0000000d, 0x0000000d, 0x3c1d0001, 0x37bde000, 0x03a0f021, 0x3c100001,
5987         0x26100000, 0x0c004010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5988         0xafbf0018, 0x0c0042e8, 0x34840002, 0x0c004364, 0x00000000, 0x3c030001,
5989         0x90630f34, 0x24020002, 0x3c040001, 0x24840e9c, 0x14620003, 0x24050001,
5990         0x3c040001, 0x24840e90, 0x24060002, 0x00003821, 0xafa00010, 0x0c004378,
5991         0xafa00014, 0x0c00402c, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5992         0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c, 0xafb20018, 0xafb10014,
5993         0x0c0042d4, 0xafb00010, 0x3c128000, 0x24110001, 0x8f706810, 0x32020400,
5994         0x10400007, 0x00000000, 0x8f641008, 0x00921024, 0x14400003, 0x00000000,
5995         0x0c004064, 0x00000000, 0x3c020001, 0x90420f56, 0x10510003, 0x32020200,
5996         0x1040fff1, 0x00000000, 0x0c0041b4, 0x00000000, 0x08004034, 0x00000000,
5997         0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020,
5998         0x27bdffe0, 0x3c040001, 0x24840eb0, 0x00002821, 0x00003021, 0x00003821,
5999         0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0000d021, 0x24020130,
6000         0xaf625000, 0x3c010001, 0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018,
6001         0x03e00008, 0x27bd0020, 0x00000000, 0x00000000, 0x3c030001, 0x24630f60,
6002         0x90620000, 0x27bdfff0, 0x14400003, 0x0080c021, 0x08004073, 0x00004821,
6003         0x3c022000, 0x03021024, 0x10400003, 0x24090002, 0x08004073, 0xa0600000,
6004         0x24090001, 0x00181040, 0x30431f80, 0x346f8008, 0x1520004b, 0x25eb0028,
6005         0x3c040001, 0x00832021, 0x8c848010, 0x3c050001, 0x24a50f7a, 0x00041402,
6006         0xa0a20000, 0x3c010001, 0xa0240f7b, 0x3c020001, 0x00431021, 0x94428014,
6007         0x3c010001, 0xa0220f7c, 0x3c0c0001, 0x01836021, 0x8d8c8018, 0x304200ff,
6008         0x24420008, 0x000220c3, 0x24020001, 0x3c010001, 0xa0220f60, 0x0124102b,
6009         0x1040000c, 0x00003821, 0x24a6000e, 0x01602821, 0x8ca20000, 0x8ca30004,
6010         0x24a50008, 0x24e70001, 0xacc20000, 0xacc30004, 0x00e4102b, 0x1440fff8,
6011         0x24c60008, 0x00003821, 0x3c080001, 0x25080f7b, 0x91060000, 0x3c020001,
6012         0x90420f7c, 0x2503000d, 0x00c32821, 0x00461023, 0x00021fc2, 0x00431021,
6013         0x00021043, 0x1840000c, 0x00002021, 0x91020001, 0x00461023, 0x00021fc2,
6014         0x00431021, 0x00021843, 0x94a20000, 0x24e70001, 0x00822021, 0x00e3102a,
6015         0x1440fffb, 0x24a50002, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
6016         0x00822021, 0x3c02ffff, 0x01821024, 0x3083ffff, 0x00431025, 0x3c010001,
6017         0x080040fa, 0xac220f80, 0x3c050001, 0x24a50f7c, 0x90a20000, 0x3c0c0001,
6018         0x01836021, 0x8d8c8018, 0x000220c2, 0x1080000e, 0x00003821, 0x01603021,
6019         0x24a5000c, 0x8ca20000, 0x8ca30004, 0x24a50008, 0x24e70001, 0xacc20000,
6020         0xacc30004, 0x00e4102b, 0x1440fff8, 0x24c60008, 0x3c050001, 0x24a50f7c,
6021         0x90a20000, 0x30430007, 0x24020004, 0x10620011, 0x28620005, 0x10400005,
6022         0x24020002, 0x10620008, 0x000710c0, 0x080040fa, 0x00000000, 0x24020006,
6023         0x1062000e, 0x000710c0, 0x080040fa, 0x00000000, 0x00a21821, 0x9463000c,
6024         0x004b1021, 0x080040fa, 0xa4430000, 0x000710c0, 0x00a21821, 0x8c63000c,
6025         0x004b1021, 0x080040fa, 0xac430000, 0x00a21821, 0x8c63000c, 0x004b2021,
6026         0x00a21021, 0xac830000, 0x94420010, 0xa4820004, 0x95e70006, 0x3c020001,
6027         0x90420f7c, 0x3c030001, 0x90630f7a, 0x00e2c823, 0x3c020001, 0x90420f7b,
6028         0x24630028, 0x01e34021, 0x24420028, 0x15200012, 0x01e23021, 0x94c2000c,
6029         0x3c010001, 0xa4220f78, 0x94c20004, 0x94c30006, 0x3c010001, 0xa4200f76,
6030         0x3c010001, 0xa4200f72, 0x00021400, 0x00431025, 0x3c010001, 0xac220f6c,
6031         0x95020004, 0x3c010001, 0x08004124, 0xa4220f70, 0x3c020001, 0x94420f70,
6032         0x3c030001, 0x94630f72, 0x00431021, 0xa5020004, 0x3c020001, 0x94420f6c,
6033         0xa4c20004, 0x3c020001, 0x8c420f6c, 0xa4c20006, 0x3c040001, 0x94840f72,
6034         0x3c020001, 0x94420f70, 0x3c0a0001, 0x954a0f76, 0x00441821, 0x3063ffff,
6035         0x0062182a, 0x24020002, 0x1122000b, 0x00832023, 0x3c030001, 0x94630f78,
6036         0x30620009, 0x10400006, 0x3062fff6, 0xa4c2000c, 0x3c020001, 0x94420f78,
6037         0x30420009, 0x01425023, 0x24020001, 0x1122001b, 0x29220002, 0x50400005,
6038         0x24020002, 0x11200007, 0x31a2ffff, 0x08004197, 0x00000000, 0x1122001d,
6039         0x24020016, 0x08004197, 0x31a2ffff, 0x3c0e0001, 0x95ce0f80, 0x10800005,
6040         0x01806821, 0x01c42021, 0x00041c02, 0x3082ffff, 0x00627021, 0x000e1027,
6041         0xa502000a, 0x3c030001, 0x90630f7b, 0x31a2ffff, 0x00e21021, 0x0800418d,
6042         0x00432023, 0x3c020001, 0x94420f80, 0x00442021, 0x00041c02, 0x3082ffff,
6043         0x00622021, 0x00807021, 0x00041027, 0x08004185, 0xa502000a, 0x3c050001,
6044         0x24a50f7a, 0x90a30000, 0x14620002, 0x24e2fff2, 0xa5e20034, 0x90a20000,
6045         0x00e21023, 0xa5020002, 0x3c030001, 0x94630f80, 0x3c020001, 0x94420f5a,
6046         0x30e5ffff, 0x00641821, 0x00451023, 0x00622023, 0x00041c02, 0x3082ffff,
6047         0x00622021, 0x00041027, 0xa502000a, 0x3c030001, 0x90630f7c, 0x24620001,
6048         0x14a20005, 0x00807021, 0x01631021, 0x90420000, 0x08004185, 0x00026200,
6049         0x24620002, 0x14a20003, 0x306200fe, 0x004b1021, 0x944c0000, 0x3c020001,
6050         0x94420f82, 0x3183ffff, 0x3c040001, 0x90840f7b, 0x00431021, 0x00e21021,
6051         0x00442023, 0x008a2021, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
6052         0x00822021, 0x00806821, 0x00041027, 0xa4c20010, 0x31a2ffff, 0x000e1c00,
6053         0x00431025, 0x3c040001, 0x24840f72, 0xade20010, 0x94820000, 0x3c050001,
6054         0x94a50f76, 0x3c030001, 0x8c630f6c, 0x24420001, 0x00b92821, 0xa4820000,
6055         0x3322ffff, 0x00622021, 0x0083182b, 0x3c010001, 0xa4250f76, 0x10600003,
6056         0x24a2ffff, 0x3c010001, 0xa4220f76, 0x3c024000, 0x03021025, 0x3c010001,
6057         0xac240f6c, 0xaf621008, 0x03e00008, 0x27bd0010, 0x3c030001, 0x90630f56,
6058         0x27bdffe8, 0x24020001, 0xafbf0014, 0x10620026, 0xafb00010, 0x8f620cf4,
6059         0x2442ffff, 0x3042007f, 0x00021100, 0x8c434000, 0x3c010001, 0xac230f64,
6060         0x8c434008, 0x24444000, 0x8c5c4004, 0x30620040, 0x14400002, 0x24020088,
6061         0x24020008, 0x3c010001, 0xa4220f68, 0x30620004, 0x10400005, 0x24020001,
6062         0x3c010001, 0xa0220f57, 0x080041d5, 0x00031402, 0x3c010001, 0xa0200f57,
6063         0x00031402, 0x3c010001, 0xa4220f54, 0x9483000c, 0x24020001, 0x3c010001,
6064         0xa4200f50, 0x3c010001, 0xa0220f56, 0x3c010001, 0xa4230f62, 0x24020001,
6065         0x1342001e, 0x00000000, 0x13400005, 0x24020003, 0x13420067, 0x00000000,
6066         0x080042cf, 0x00000000, 0x3c020001, 0x94420f62, 0x241a0001, 0x3c010001,
6067         0xa4200f5e, 0x3c010001, 0xa4200f52, 0x304407ff, 0x00021bc2, 0x00031823,
6068         0x3063003e, 0x34630036, 0x00021242, 0x3042003c, 0x00621821, 0x3c010001,
6069         0xa4240f58, 0x00832021, 0x24630030, 0x3c010001, 0xa4240f5a, 0x3c010001,
6070         0xa4230f5c, 0x3c060001, 0x24c60f52, 0x94c50000, 0x94c30002, 0x3c040001,
6071         0x94840f5a, 0x00651021, 0x0044102a, 0x10400013, 0x3c108000, 0x00a31021,
6072         0xa4c20000, 0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008,
6073         0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4,
6074         0x00501024, 0x104000b7, 0x00000000, 0x0800420f, 0x00000000, 0x3c030001,
6075         0x94630f50, 0x00851023, 0xa4c40000, 0x00621821, 0x3042ffff, 0x3c010001,
6076         0xa4230f50, 0xaf620ce8, 0x3c020001, 0x94420f68, 0x34420024, 0xaf620cec,
6077         0x94c30002, 0x3c020001, 0x94420f50, 0x14620012, 0x3c028000, 0x3c108000,
6078         0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008, 0x00901024,
6079         0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024,
6080         0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003, 0xaf620cf4, 0x3c108000,
6081         0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000,
6082         0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003,
6083         0x3c070001, 0x24e70f50, 0x94e20000, 0x03821021, 0xaf620ce0, 0x3c020001,
6084         0x8c420f64, 0xaf620ce4, 0x3c050001, 0x94a50f54, 0x94e30000, 0x3c040001,
6085         0x94840f58, 0x3c020001, 0x94420f5e, 0x00a32823, 0x00822023, 0x30a6ffff,
6086         0x3083ffff, 0x00c3102b, 0x14400043, 0x00000000, 0x3c020001, 0x94420f5c,
6087         0x00021400, 0x00621025, 0xaf620ce8, 0x94e20000, 0x3c030001, 0x94630f54,
6088         0x00441021, 0xa4e20000, 0x3042ffff, 0x14430021, 0x3c020008, 0x3c020001,
6089         0x90420f57, 0x10400006, 0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624,
6090         0x0800427c, 0x0000d021, 0x3c020001, 0x94420f68, 0x3c030008, 0x34630624,
6091         0x00431025, 0xaf620cec, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
6092         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
6093         0x00000000, 0x8f620cf4, 0x00501024, 0x10400015, 0x00000000, 0x08004283,
6094         0x00000000, 0x3c030001, 0x94630f68, 0x34420624, 0x3c108000, 0x00621825,
6095         0x3c028000, 0xaf630cec, 0xaf620cf4, 0x8f641008, 0x00901024, 0x14400003,
6096         0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7,
6097         0x00000000, 0x3c010001, 0x080042cf, 0xa4200f5e, 0x3c020001, 0x94420f5c,
6098         0x00021400, 0x00c21025, 0xaf620ce8, 0x3c020001, 0x90420f57, 0x10400009,
6099         0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624, 0x0000d021, 0x00431025,
6100         0xaf620cec, 0x080042c1, 0x3c108000, 0x3c020001, 0x94420f68, 0x3c030008,
6101         0x34630604, 0x00431025, 0xaf620cec, 0x3c020001, 0x94420f5e, 0x00451021,
6102         0x3c010001, 0xa4220f5e, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
6103         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
6104         0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x8fbf0014,
6105         0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000, 0x27bdffe0, 0x3c040001,
6106         0x24840ec0, 0x00002821, 0x00003021, 0x00003821, 0xafbf0018, 0xafa00010,
6107         0x0c004378, 0xafa00014, 0x0000d021, 0x24020130, 0xaf625000, 0x3c010001,
6108         0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018, 0x03e00008, 0x27bd0020,
6109         0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010, 0xaf60680c, 0x8f626804,
6110         0x34420082, 0xaf626804, 0x8f634000, 0x24020b50, 0x3c010001, 0xac220f20,
6111         0x24020b78, 0x3c010001, 0xac220f30, 0x34630002, 0xaf634000, 0x0c004315,
6112         0x00808021, 0x3c010001, 0xa0220f34, 0x304200ff, 0x24030002, 0x14430005,
6113         0x00000000, 0x3c020001, 0x8c420f20, 0x08004308, 0xac5000c0, 0x3c020001,
6114         0x8c420f20, 0xac5000bc, 0x8f624434, 0x8f634438, 0x8f644410, 0x3c010001,
6115         0xac220f28, 0x3c010001, 0xac230f38, 0x3c010001, 0xac240f24, 0x8fbf0014,
6116         0x8fb00010, 0x03e00008, 0x27bd0018, 0x03e00008, 0x24020001, 0x27bdfff8,
6117         0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe, 0x00000000,
6118         0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008, 0x27bd0008,
6119         0x8f634450, 0x3c020001, 0x8c420f28, 0x00031c02, 0x0043102b, 0x14400008,
6120         0x3c038000, 0x3c040001, 0x8c840f38, 0x8f624450, 0x00021c02, 0x0083102b,
6121         0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024, 0x1440fffd,
6122         0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff, 0x2442e000,
6123         0x2c422001, 0x14400003, 0x3c024000, 0x08004347, 0x2402ffff, 0x00822025,
6124         0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021, 0x03e00008,
6125         0x00000000, 0x8f624450, 0x3c030001, 0x8c630f24, 0x08004350, 0x3042ffff,
6126         0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000, 0x03e00008,
6127         0x00000000, 0x27bdffe0, 0x00802821, 0x3c040001, 0x24840ed0, 0x00003021,
6128         0x00003821, 0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0800435f,
6129         0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x3c020001, 0x3442d600,
6130         0x3c030001, 0x3463d600, 0x3c040001, 0x3484ddff, 0x3c010001, 0xac220f40,
6131         0x24020040, 0x3c010001, 0xac220f44, 0x3c010001, 0xac200f3c, 0xac600000,
6132         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
6133         0x00804821, 0x8faa0010, 0x3c020001, 0x8c420f3c, 0x3c040001, 0x8c840f44,
6134         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010001, 0xac230f3c, 0x14400003,
6135         0x00004021, 0x3c010001, 0xac200f3c, 0x3c020001, 0x8c420f3c, 0x3c030001,
6136         0x8c630f40, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
6137         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020001, 0x8c420f3c,
6138         0x3c030001, 0x8c630f40, 0x8f64680c, 0x00021140, 0x00431021, 0xac440008,
6139         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
6140         0x00000000, 0x00000000, 0x00000000,
6141 };
6142
6143 static const u32 tg3Tso5FwRodata[(TG3_TSO5_FW_RODATA_LEN / 4) + 1] = {
6144         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
6145         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x00000000, 0x00000000,
6146         0x73746b6f, 0x66666c64, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
6147         0x00000000, 0x00000000, 0x00000000,
6148 };
6149
6150 static const u32 tg3Tso5FwData[(TG3_TSO5_FW_DATA_LEN / 4) + 1] = {
6151         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x322e3000, 0x00000000,
6152         0x00000000, 0x00000000, 0x00000000,
6153 };
6154
6155 /* tp->lock is held. */
6156 static int tg3_load_tso_firmware(struct tg3 *tp)
6157 {
6158         struct fw_info info;
6159         unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
6160         int err, i;
6161
6162         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6163                 return 0;
6164
6165         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
6166                 info.text_base = TG3_TSO5_FW_TEXT_ADDR;
6167                 info.text_len = TG3_TSO5_FW_TEXT_LEN;
6168                 info.text_data = &tg3Tso5FwText[0];
6169                 info.rodata_base = TG3_TSO5_FW_RODATA_ADDR;
6170                 info.rodata_len = TG3_TSO5_FW_RODATA_LEN;
6171                 info.rodata_data = &tg3Tso5FwRodata[0];
6172                 info.data_base = TG3_TSO5_FW_DATA_ADDR;
6173                 info.data_len = TG3_TSO5_FW_DATA_LEN;
6174                 info.data_data = &tg3Tso5FwData[0];
6175                 cpu_base = RX_CPU_BASE;
6176                 cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
6177                 cpu_scratch_size = (info.text_len +
6178                                     info.rodata_len +
6179                                     info.data_len +
6180                                     TG3_TSO5_FW_SBSS_LEN +
6181                                     TG3_TSO5_FW_BSS_LEN);
6182         } else {
6183                 info.text_base = TG3_TSO_FW_TEXT_ADDR;
6184                 info.text_len = TG3_TSO_FW_TEXT_LEN;
6185                 info.text_data = &tg3TsoFwText[0];
6186                 info.rodata_base = TG3_TSO_FW_RODATA_ADDR;
6187                 info.rodata_len = TG3_TSO_FW_RODATA_LEN;
6188                 info.rodata_data = &tg3TsoFwRodata[0];
6189                 info.data_base = TG3_TSO_FW_DATA_ADDR;
6190                 info.data_len = TG3_TSO_FW_DATA_LEN;
6191                 info.data_data = &tg3TsoFwData[0];
6192                 cpu_base = TX_CPU_BASE;
6193                 cpu_scratch_base = TX_CPU_SCRATCH_BASE;
6194                 cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
6195         }
6196
6197         err = tg3_load_firmware_cpu(tp, cpu_base,
6198                                     cpu_scratch_base, cpu_scratch_size,
6199                                     &info);
6200         if (err)
6201                 return err;
6202
6203         /* Now startup the cpu. */
6204         tw32(cpu_base + CPU_STATE, 0xffffffff);
6205         tw32_f(cpu_base + CPU_PC,    info.text_base);
6206
6207         for (i = 0; i < 5; i++) {
6208                 if (tr32(cpu_base + CPU_PC) == info.text_base)
6209                         break;
6210                 tw32(cpu_base + CPU_STATE, 0xffffffff);
6211                 tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
6212                 tw32_f(cpu_base + CPU_PC,    info.text_base);
6213                 udelay(1000);
6214         }
6215         if (i >= 5) {
6216                 printk(KERN_ERR PFX "tg3_load_tso_firmware fails for %s "
6217                        "to set CPU PC, is %08x should be %08x\n",
6218                        tp->dev->name, tr32(cpu_base + CPU_PC),
6219                        info.text_base);
6220                 return -ENODEV;
6221         }
6222         tw32(cpu_base + CPU_STATE, 0xffffffff);
6223         tw32_f(cpu_base + CPU_MODE,  0x00000000);
6224         return 0;
6225 }
6226
6227
6228 /* tp->lock is held. */
6229 static void __tg3_set_mac_addr(struct tg3 *tp, int skip_mac_1)
6230 {
6231         u32 addr_high, addr_low;
6232         int i;
6233
6234         addr_high = ((tp->dev->dev_addr[0] << 8) |
6235                      tp->dev->dev_addr[1]);
6236         addr_low = ((tp->dev->dev_addr[2] << 24) |
6237                     (tp->dev->dev_addr[3] << 16) |
6238                     (tp->dev->dev_addr[4] <<  8) |
6239                     (tp->dev->dev_addr[5] <<  0));
6240         for (i = 0; i < 4; i++) {
6241                 if (i == 1 && skip_mac_1)
6242                         continue;
6243                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
6244                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
6245         }
6246
6247         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
6248             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
6249                 for (i = 0; i < 12; i++) {
6250                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
6251                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
6252                 }
6253         }
6254
6255         addr_high = (tp->dev->dev_addr[0] +
6256                      tp->dev->dev_addr[1] +
6257                      tp->dev->dev_addr[2] +
6258                      tp->dev->dev_addr[3] +
6259                      tp->dev->dev_addr[4] +
6260                      tp->dev->dev_addr[5]) &
6261                 TX_BACKOFF_SEED_MASK;
6262         tw32(MAC_TX_BACKOFF_SEED, addr_high);
6263 }
6264
6265 static int tg3_set_mac_addr(struct net_device *dev, void *p)
6266 {
6267         struct tg3 *tp = netdev_priv(dev);
6268         struct sockaddr *addr = p;
6269         int err = 0, skip_mac_1 = 0;
6270
6271         if (!is_valid_ether_addr(addr->sa_data))
6272                 return -EINVAL;
6273
6274         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
6275
6276         if (!netif_running(dev))
6277                 return 0;
6278
6279         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
6280                 u32 addr0_high, addr0_low, addr1_high, addr1_low;
6281
6282                 addr0_high = tr32(MAC_ADDR_0_HIGH);
6283                 addr0_low = tr32(MAC_ADDR_0_LOW);
6284                 addr1_high = tr32(MAC_ADDR_1_HIGH);
6285                 addr1_low = tr32(MAC_ADDR_1_LOW);
6286
6287                 /* Skip MAC addr 1 if ASF is using it. */
6288                 if ((addr0_high != addr1_high || addr0_low != addr1_low) &&
6289                     !(addr1_high == 0 && addr1_low == 0))
6290                         skip_mac_1 = 1;
6291         }
6292         spin_lock_bh(&tp->lock);
6293         __tg3_set_mac_addr(tp, skip_mac_1);
6294         spin_unlock_bh(&tp->lock);
6295
6296         return err;
6297 }
6298
6299 /* tp->lock is held. */
6300 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
6301                            dma_addr_t mapping, u32 maxlen_flags,
6302                            u32 nic_addr)
6303 {
6304         tg3_write_mem(tp,
6305                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
6306                       ((u64) mapping >> 32));
6307         tg3_write_mem(tp,
6308                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
6309                       ((u64) mapping & 0xffffffff));
6310         tg3_write_mem(tp,
6311                       (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
6312                        maxlen_flags);
6313
6314         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6315                 tg3_write_mem(tp,
6316                               (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
6317                               nic_addr);
6318 }
6319
6320 static void __tg3_set_rx_mode(struct net_device *);
6321 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
6322 {
6323         tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
6324         tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
6325         tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
6326         tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
6327         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6328                 tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
6329                 tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
6330         }
6331         tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
6332         tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
6333         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6334                 u32 val = ec->stats_block_coalesce_usecs;
6335
6336                 if (!netif_carrier_ok(tp->dev))
6337                         val = 0;
6338
6339                 tw32(HOSTCC_STAT_COAL_TICKS, val);
6340         }
6341 }
6342
6343 /* tp->lock is held. */
6344 static int tg3_reset_hw(struct tg3 *tp, int reset_phy)
6345 {
6346         u32 val, rdmac_mode;
6347         int i, err, limit;
6348
6349         tg3_disable_ints(tp);
6350
6351         tg3_stop_fw(tp);
6352
6353         tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
6354
6355         if (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) {
6356                 tg3_abort_hw(tp, 1);
6357         }
6358
6359         if (reset_phy)
6360                 tg3_phy_reset(tp);
6361
6362         err = tg3_chip_reset(tp);
6363         if (err)
6364                 return err;
6365
6366         tg3_write_sig_legacy(tp, RESET_KIND_INIT);
6367
6368         if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0) {
6369                 val = tr32(TG3_CPMU_CTRL);
6370                 val &= ~(CPMU_CTRL_LINK_AWARE_MODE | CPMU_CTRL_LINK_IDLE_MODE);
6371                 tw32(TG3_CPMU_CTRL, val);
6372
6373                 val = tr32(TG3_CPMU_LSPD_10MB_CLK);
6374                 val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
6375                 val |= CPMU_LSPD_10MB_MACCLK_6_25;
6376                 tw32(TG3_CPMU_LSPD_10MB_CLK, val);
6377
6378                 val = tr32(TG3_CPMU_LNK_AWARE_PWRMD);
6379                 val &= ~CPMU_LNK_AWARE_MACCLK_MASK;
6380                 val |= CPMU_LNK_AWARE_MACCLK_6_25;
6381                 tw32(TG3_CPMU_LNK_AWARE_PWRMD, val);
6382
6383                 val = tr32(TG3_CPMU_HST_ACC);
6384                 val &= ~CPMU_HST_ACC_MACCLK_MASK;
6385                 val |= CPMU_HST_ACC_MACCLK_6_25;
6386                 tw32(TG3_CPMU_HST_ACC, val);
6387         }
6388
6389         /* This works around an issue with Athlon chipsets on
6390          * B3 tigon3 silicon.  This bit has no effect on any
6391          * other revision.  But do not set this on PCI Express
6392          * chips and don't even touch the clocks if the CPMU is present.
6393          */
6394         if (!(tp->tg3_flags & TG3_FLAG_CPMU_PRESENT)) {
6395                 if (!(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
6396                         tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
6397                 tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
6398         }
6399
6400         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
6401             (tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
6402                 val = tr32(TG3PCI_PCISTATE);
6403                 val |= PCISTATE_RETRY_SAME_DMA;
6404                 tw32(TG3PCI_PCISTATE, val);
6405         }
6406
6407         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
6408                 /* Allow reads and writes to the
6409                  * APE register and memory space.
6410                  */
6411                 val = tr32(TG3PCI_PCISTATE);
6412                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
6413                        PCISTATE_ALLOW_APE_SHMEM_WR;
6414                 tw32(TG3PCI_PCISTATE, val);
6415         }
6416
6417         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_BX) {
6418                 /* Enable some hw fixes.  */
6419                 val = tr32(TG3PCI_MSI_DATA);
6420                 val |= (1 << 26) | (1 << 28) | (1 << 29);
6421                 tw32(TG3PCI_MSI_DATA, val);
6422         }
6423
6424         /* Descriptor ring init may make accesses to the
6425          * NIC SRAM area to setup the TX descriptors, so we
6426          * can only do this after the hardware has been
6427          * successfully reset.
6428          */
6429         err = tg3_init_rings(tp);
6430         if (err)
6431                 return err;
6432
6433         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 &&
6434             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761) {
6435                 /* This value is determined during the probe time DMA
6436                  * engine test, tg3_test_dma.
6437                  */
6438                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
6439         }
6440
6441         tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
6442                           GRC_MODE_4X_NIC_SEND_RINGS |
6443                           GRC_MODE_NO_TX_PHDR_CSUM |
6444                           GRC_MODE_NO_RX_PHDR_CSUM);
6445         tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
6446
6447         /* Pseudo-header checksum is done by hardware logic and not
6448          * the offload processers, so make the chip do the pseudo-
6449          * header checksums on receive.  For transmit it is more
6450          * convenient to do the pseudo-header checksum in software
6451          * as Linux does that on transmit for us in all cases.
6452          */
6453         tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
6454
6455         tw32(GRC_MODE,
6456              tp->grc_mode |
6457              (GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP));
6458
6459         /* Setup the timer prescalar register.  Clock is always 66Mhz. */
6460         val = tr32(GRC_MISC_CFG);
6461         val &= ~0xff;
6462         val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
6463         tw32(GRC_MISC_CFG, val);
6464
6465         /* Initialize MBUF/DESC pool. */
6466         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
6467                 /* Do nothing.  */
6468         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705) {
6469                 tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
6470                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
6471                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
6472                 else
6473                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
6474                 tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
6475                 tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
6476         }
6477         else if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
6478                 int fw_len;
6479
6480                 fw_len = (TG3_TSO5_FW_TEXT_LEN +
6481                           TG3_TSO5_FW_RODATA_LEN +
6482                           TG3_TSO5_FW_DATA_LEN +
6483                           TG3_TSO5_FW_SBSS_LEN +
6484                           TG3_TSO5_FW_BSS_LEN);
6485                 fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
6486                 tw32(BUFMGR_MB_POOL_ADDR,
6487                      NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
6488                 tw32(BUFMGR_MB_POOL_SIZE,
6489                      NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
6490         }
6491
6492         if (tp->dev->mtu <= ETH_DATA_LEN) {
6493                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
6494                      tp->bufmgr_config.mbuf_read_dma_low_water);
6495                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
6496                      tp->bufmgr_config.mbuf_mac_rx_low_water);
6497                 tw32(BUFMGR_MB_HIGH_WATER,
6498                      tp->bufmgr_config.mbuf_high_water);
6499         } else {
6500                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
6501                      tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
6502                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
6503                      tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
6504                 tw32(BUFMGR_MB_HIGH_WATER,
6505                      tp->bufmgr_config.mbuf_high_water_jumbo);
6506         }
6507         tw32(BUFMGR_DMA_LOW_WATER,
6508              tp->bufmgr_config.dma_low_water);
6509         tw32(BUFMGR_DMA_HIGH_WATER,
6510              tp->bufmgr_config.dma_high_water);
6511
6512         tw32(BUFMGR_MODE, BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE);
6513         for (i = 0; i < 2000; i++) {
6514                 if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
6515                         break;
6516                 udelay(10);
6517         }
6518         if (i >= 2000) {
6519                 printk(KERN_ERR PFX "tg3_reset_hw cannot enable BUFMGR for %s.\n",
6520                        tp->dev->name);
6521                 return -ENODEV;
6522         }
6523
6524         /* Setup replenish threshold. */
6525         val = tp->rx_pending / 8;
6526         if (val == 0)
6527                 val = 1;
6528         else if (val > tp->rx_std_max_post)
6529                 val = tp->rx_std_max_post;
6530         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
6531                 if (tp->pci_chip_rev_id == CHIPREV_ID_5906_A1)
6532                         tw32(ISO_PKT_TX, (tr32(ISO_PKT_TX) & ~0x3) | 0x2);
6533
6534                 if (val > (TG3_RX_INTERNAL_RING_SZ_5906 / 2))
6535                         val = TG3_RX_INTERNAL_RING_SZ_5906 / 2;
6536         }
6537
6538         tw32(RCVBDI_STD_THRESH, val);
6539
6540         /* Initialize TG3_BDINFO's at:
6541          *  RCVDBDI_STD_BD:     standard eth size rx ring
6542          *  RCVDBDI_JUMBO_BD:   jumbo frame rx ring
6543          *  RCVDBDI_MINI_BD:    small frame rx ring (??? does not work)
6544          *
6545          * like so:
6546          *  TG3_BDINFO_HOST_ADDR:       high/low parts of DMA address of ring
6547          *  TG3_BDINFO_MAXLEN_FLAGS:    (rx max buffer size << 16) |
6548          *                              ring attribute flags
6549          *  TG3_BDINFO_NIC_ADDR:        location of descriptors in nic SRAM
6550          *
6551          * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
6552          * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
6553          *
6554          * The size of each ring is fixed in the firmware, but the location is
6555          * configurable.
6556          */
6557         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6558              ((u64) tp->rx_std_mapping >> 32));
6559         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6560              ((u64) tp->rx_std_mapping & 0xffffffff));
6561         tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
6562              NIC_SRAM_RX_BUFFER_DESC);
6563
6564         /* Don't even try to program the JUMBO/MINI buffer descriptor
6565          * configs on 5705.
6566          */
6567         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
6568                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6569                      RX_STD_MAX_SIZE_5705 << BDINFO_FLAGS_MAXLEN_SHIFT);
6570         } else {
6571                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6572                      RX_STD_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6573
6574                 tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
6575                      BDINFO_FLAGS_DISABLED);
6576
6577                 /* Setup replenish threshold. */
6578                 tw32(RCVBDI_JUMBO_THRESH, tp->rx_jumbo_pending / 8);
6579
6580                 if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
6581                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6582                              ((u64) tp->rx_jumbo_mapping >> 32));
6583                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6584                              ((u64) tp->rx_jumbo_mapping & 0xffffffff));
6585                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6586                              RX_JUMBO_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6587                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
6588                              NIC_SRAM_RX_JUMBO_BUFFER_DESC);
6589                 } else {
6590                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6591                              BDINFO_FLAGS_DISABLED);
6592                 }
6593
6594         }
6595
6596         /* There is only one send ring on 5705/5750, no need to explicitly
6597          * disable the others.
6598          */
6599         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6600                 /* Clear out send RCB ring in SRAM. */
6601                 for (i = NIC_SRAM_SEND_RCB; i < NIC_SRAM_RCV_RET_RCB; i += TG3_BDINFO_SIZE)
6602                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6603                                       BDINFO_FLAGS_DISABLED);
6604         }
6605
6606         tp->tx_prod = 0;
6607         tp->tx_cons = 0;
6608         tw32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6609         tw32_tx_mbox(MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6610
6611         tg3_set_bdinfo(tp, NIC_SRAM_SEND_RCB,
6612                        tp->tx_desc_mapping,
6613                        (TG3_TX_RING_SIZE <<
6614                         BDINFO_FLAGS_MAXLEN_SHIFT),
6615                        NIC_SRAM_TX_BUFFER_DESC);
6616
6617         /* There is only one receive return ring on 5705/5750, no need
6618          * to explicitly disable the others.
6619          */
6620         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6621                 for (i = NIC_SRAM_RCV_RET_RCB; i < NIC_SRAM_STATS_BLK;
6622                      i += TG3_BDINFO_SIZE) {
6623                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6624                                       BDINFO_FLAGS_DISABLED);
6625                 }
6626         }
6627
6628         tp->rx_rcb_ptr = 0;
6629         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, 0);
6630
6631         tg3_set_bdinfo(tp, NIC_SRAM_RCV_RET_RCB,
6632                        tp->rx_rcb_mapping,
6633                        (TG3_RX_RCB_RING_SIZE(tp) <<
6634                         BDINFO_FLAGS_MAXLEN_SHIFT),
6635                        0);
6636
6637         tp->rx_std_ptr = tp->rx_pending;
6638         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
6639                      tp->rx_std_ptr);
6640
6641         tp->rx_jumbo_ptr = (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) ?
6642                                                 tp->rx_jumbo_pending : 0;
6643         tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
6644                      tp->rx_jumbo_ptr);
6645
6646         /* Initialize MAC address and backoff seed. */
6647         __tg3_set_mac_addr(tp, 0);
6648
6649         /* MTU + ethernet header + FCS + optional VLAN tag */
6650         tw32(MAC_RX_MTU_SIZE, tp->dev->mtu + ETH_HLEN + 8);
6651
6652         /* The slot time is changed by tg3_setup_phy if we
6653          * run at gigabit with half duplex.
6654          */
6655         tw32(MAC_TX_LENGTHS,
6656              (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
6657              (6 << TX_LENGTHS_IPG_SHIFT) |
6658              (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
6659
6660         /* Receive rules. */
6661         tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
6662         tw32(RCVLPC_CONFIG, 0x0181);
6663
6664         /* Calculate RDMAC_MODE setting early, we need it to determine
6665          * the RCVLPC_STATE_ENABLE mask.
6666          */
6667         rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
6668                       RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
6669                       RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
6670                       RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
6671                       RDMAC_MODE_LNGREAD_ENAB);
6672
6673         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784)
6674                 rdmac_mode |= RDMAC_MODE_BD_SBD_CRPT_ENAB |
6675                               RDMAC_MODE_MBUF_RBD_CRPT_ENAB |
6676                               RDMAC_MODE_MBUF_SBD_CRPT_ENAB;
6677
6678         /* If statement applies to 5705 and 5750 PCI devices only */
6679         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6680              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6681             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)) {
6682                 if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE &&
6683                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
6684                         rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
6685                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6686                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788)) {
6687                         rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6688                 }
6689         }
6690
6691         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
6692                 rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6693
6694         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6695                 rdmac_mode |= (1 << 27);
6696
6697         /* Receive/send statistics. */
6698         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
6699                 val = tr32(RCVLPC_STATS_ENABLE);
6700                 val &= ~RCVLPC_STATSENAB_DACK_FIX;
6701                 tw32(RCVLPC_STATS_ENABLE, val);
6702         } else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
6703                    (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
6704                 val = tr32(RCVLPC_STATS_ENABLE);
6705                 val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
6706                 tw32(RCVLPC_STATS_ENABLE, val);
6707         } else {
6708                 tw32(RCVLPC_STATS_ENABLE, 0xffffff);
6709         }
6710         tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
6711         tw32(SNDDATAI_STATSENAB, 0xffffff);
6712         tw32(SNDDATAI_STATSCTRL,
6713              (SNDDATAI_SCTRL_ENABLE |
6714               SNDDATAI_SCTRL_FASTUPD));
6715
6716         /* Setup host coalescing engine. */
6717         tw32(HOSTCC_MODE, 0);
6718         for (i = 0; i < 2000; i++) {
6719                 if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
6720                         break;
6721                 udelay(10);
6722         }
6723
6724         __tg3_set_coalesce(tp, &tp->coal);
6725
6726         /* set status block DMA address */
6727         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6728              ((u64) tp->status_mapping >> 32));
6729         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6730              ((u64) tp->status_mapping & 0xffffffff));
6731
6732         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6733                 /* Status/statistics block address.  See tg3_timer,
6734                  * the tg3_periodic_fetch_stats call there, and
6735                  * tg3_get_stats to see how this works for 5705/5750 chips.
6736                  */
6737                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6738                      ((u64) tp->stats_mapping >> 32));
6739                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6740                      ((u64) tp->stats_mapping & 0xffffffff));
6741                 tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
6742                 tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
6743         }
6744
6745         tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
6746
6747         tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
6748         tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
6749         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6750                 tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
6751
6752         /* Clear statistics/status block in chip, and status block in ram. */
6753         for (i = NIC_SRAM_STATS_BLK;
6754              i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
6755              i += sizeof(u32)) {
6756                 tg3_write_mem(tp, i, 0);
6757                 udelay(40);
6758         }
6759         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
6760
6761         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
6762                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
6763                 /* reset to prevent losing 1st rx packet intermittently */
6764                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
6765                 udelay(10);
6766         }
6767
6768         tp->mac_mode = MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
6769                 MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE | MAC_MODE_FHDE_ENABLE;
6770         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
6771             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
6772             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700)
6773                 tp->mac_mode |= MAC_MODE_LINK_POLARITY;
6774         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
6775         udelay(40);
6776
6777         /* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
6778          * If TG3_FLG2_IS_NIC is zero, we should read the
6779          * register to preserve the GPIO settings for LOMs. The GPIOs,
6780          * whether used as inputs or outputs, are set by boot code after
6781          * reset.
6782          */
6783         if (!(tp->tg3_flags2 & TG3_FLG2_IS_NIC)) {
6784                 u32 gpio_mask;
6785
6786                 gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE1 |
6787                             GRC_LCLCTRL_GPIO_OE2 | GRC_LCLCTRL_GPIO_OUTPUT0 |
6788                             GRC_LCLCTRL_GPIO_OUTPUT1 | GRC_LCLCTRL_GPIO_OUTPUT2;
6789
6790                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
6791                         gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
6792                                      GRC_LCLCTRL_GPIO_OUTPUT3;
6793
6794                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
6795                         gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
6796
6797                 tp->grc_local_ctrl &= ~gpio_mask;
6798                 tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
6799
6800                 /* GPIO1 must be driven high for eeprom write protect */
6801                 if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT)
6802                         tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
6803                                                GRC_LCLCTRL_GPIO_OUTPUT1);
6804         }
6805         tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
6806         udelay(100);
6807
6808         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0);
6809         tp->last_tag = 0;
6810
6811         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6812                 tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
6813                 udelay(40);
6814         }
6815
6816         val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
6817                WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
6818                WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
6819                WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
6820                WDMAC_MODE_LNGREAD_ENAB);
6821
6822         /* If statement applies to 5705 and 5750 PCI devices only */
6823         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6824              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6825             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) {
6826                 if ((tp->tg3_flags & TG3_FLG2_TSO_CAPABLE) &&
6827                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
6828                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
6829                         /* nothing */
6830                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6831                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
6832                            !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
6833                         val |= WDMAC_MODE_RX_ACCEL;
6834                 }
6835         }
6836
6837         /* Enable host coalescing bug fix */
6838         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755) ||
6839             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787) ||
6840             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784) ||
6841             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761))
6842                 val |= (1 << 29);
6843
6844         tw32_f(WDMAC_MODE, val);
6845         udelay(40);
6846
6847         if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
6848                 u16 pcix_cmd;
6849
6850                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
6851                                      &pcix_cmd);
6852                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703) {
6853                         pcix_cmd &= ~PCI_X_CMD_MAX_READ;
6854                         pcix_cmd |= PCI_X_CMD_READ_2K;
6855                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
6856                         pcix_cmd &= ~(PCI_X_CMD_MAX_SPLIT | PCI_X_CMD_MAX_READ);
6857                         pcix_cmd |= PCI_X_CMD_READ_2K;
6858                 }
6859                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
6860                                       pcix_cmd);
6861         }
6862
6863         tw32_f(RDMAC_MODE, rdmac_mode);
6864         udelay(40);
6865
6866         tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
6867         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6868                 tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
6869
6870         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
6871                 tw32(SNDDATAC_MODE,
6872                      SNDDATAC_MODE_ENABLE | SNDDATAC_MODE_CDELAY);
6873         else
6874                 tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
6875
6876         tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
6877         tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
6878         tw32(RCVDBDI_MODE, RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ);
6879         tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
6880         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6881                 tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
6882         tw32(SNDBDI_MODE, SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE);
6883         tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
6884
6885         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
6886                 err = tg3_load_5701_a0_firmware_fix(tp);
6887                 if (err)
6888                         return err;
6889         }
6890
6891         if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
6892                 err = tg3_load_tso_firmware(tp);
6893                 if (err)
6894                         return err;
6895         }
6896
6897         tp->tx_mode = TX_MODE_ENABLE;
6898         tw32_f(MAC_TX_MODE, tp->tx_mode);
6899         udelay(100);
6900
6901         tp->rx_mode = RX_MODE_ENABLE;
6902         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
6903             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
6904                 tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
6905
6906         tw32_f(MAC_RX_MODE, tp->rx_mode);
6907         udelay(10);
6908
6909         if (tp->link_config.phy_is_low_power) {
6910                 tp->link_config.phy_is_low_power = 0;
6911                 tp->link_config.speed = tp->link_config.orig_speed;
6912                 tp->link_config.duplex = tp->link_config.orig_duplex;
6913                 tp->link_config.autoneg = tp->link_config.orig_autoneg;
6914         }
6915
6916         tp->mi_mode = MAC_MI_MODE_BASE;
6917         tw32_f(MAC_MI_MODE, tp->mi_mode);
6918         udelay(80);
6919
6920         tw32(MAC_LED_CTRL, tp->led_ctrl);
6921
6922         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
6923         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
6924                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
6925                 udelay(10);
6926         }
6927         tw32_f(MAC_RX_MODE, tp->rx_mode);
6928         udelay(10);
6929
6930         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
6931                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) &&
6932                         !(tp->tg3_flags2 & TG3_FLG2_SERDES_PREEMPHASIS)) {
6933                         /* Set drive transmission level to 1.2V  */
6934                         /* only if the signal pre-emphasis bit is not set  */
6935                         val = tr32(MAC_SERDES_CFG);
6936                         val &= 0xfffff000;
6937                         val |= 0x880;
6938                         tw32(MAC_SERDES_CFG, val);
6939                 }
6940                 if (tp->pci_chip_rev_id == CHIPREV_ID_5703_A1)
6941                         tw32(MAC_SERDES_CFG, 0x616000);
6942         }
6943
6944         /* Prevent chip from dropping frames when flow control
6945          * is enabled.
6946          */
6947         tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, 2);
6948
6949         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
6950             (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
6951                 /* Use hardware link auto-negotiation */
6952                 tp->tg3_flags2 |= TG3_FLG2_HW_AUTONEG;
6953         }
6954
6955         if ((tp->tg3_flags2 & TG3_FLG2_MII_SERDES) &&
6956             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
6957                 u32 tmp;
6958
6959                 tmp = tr32(SERDES_RX_CTRL);
6960                 tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
6961                 tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
6962                 tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
6963                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
6964         }
6965
6966         err = tg3_setup_phy(tp, 0);
6967         if (err)
6968                 return err;
6969
6970         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
6971             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906) {
6972                 u32 tmp;
6973
6974                 /* Clear CRC stats. */
6975                 if (!tg3_readphy(tp, MII_TG3_TEST1, &tmp)) {
6976                         tg3_writephy(tp, MII_TG3_TEST1,
6977                                      tmp | MII_TG3_TEST1_CRC_EN);
6978                         tg3_readphy(tp, 0x14, &tmp);
6979                 }
6980         }
6981
6982         __tg3_set_rx_mode(tp->dev);
6983
6984         /* Initialize receive rules. */
6985         tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
6986         tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
6987         tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
6988         tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
6989
6990         if ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
6991             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
6992                 limit = 8;
6993         else
6994                 limit = 16;
6995         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF)
6996                 limit -= 4;
6997         switch (limit) {
6998         case 16:
6999                 tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
7000         case 15:
7001                 tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
7002         case 14:
7003                 tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
7004         case 13:
7005                 tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
7006         case 12:
7007                 tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
7008         case 11:
7009                 tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
7010         case 10:
7011                 tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
7012         case 9:
7013                 tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
7014         case 8:
7015                 tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
7016         case 7:
7017                 tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
7018         case 6:
7019                 tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
7020         case 5:
7021                 tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
7022         case 4:
7023                 /* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
7024         case 3:
7025                 /* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
7026         case 2:
7027         case 1:
7028
7029         default:
7030                 break;
7031         };
7032
7033         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
7034                 /* Write our heartbeat update interval to APE. */
7035                 tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
7036                                 APE_HOST_HEARTBEAT_INT_DISABLE);
7037
7038         tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
7039
7040         return 0;
7041 }
7042
7043 /* Called at device open time to get the chip ready for
7044  * packet processing.  Invoked with tp->lock held.
7045  */
7046 static int tg3_init_hw(struct tg3 *tp, int reset_phy)
7047 {
7048         int err;
7049
7050         /* Force the chip into D0. */
7051         err = tg3_set_power_state(tp, PCI_D0);
7052         if (err)
7053                 goto out;
7054
7055         tg3_switch_clocks(tp);
7056
7057         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
7058
7059         err = tg3_reset_hw(tp, reset_phy);
7060
7061 out:
7062         return err;
7063 }
7064
7065 #define TG3_STAT_ADD32(PSTAT, REG) \
7066 do {    u32 __val = tr32(REG); \
7067         (PSTAT)->low += __val; \
7068         if ((PSTAT)->low < __val) \
7069                 (PSTAT)->high += 1; \
7070 } while (0)
7071
7072 static void tg3_periodic_fetch_stats(struct tg3 *tp)
7073 {
7074         struct tg3_hw_stats *sp = tp->hw_stats;
7075
7076         if (!netif_carrier_ok(tp->dev))
7077                 return;
7078
7079         TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
7080         TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
7081         TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
7082         TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
7083         TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
7084         TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
7085         TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
7086         TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
7087         TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
7088         TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
7089         TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
7090         TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
7091         TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
7092
7093         TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
7094         TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
7095         TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
7096         TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
7097         TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
7098         TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
7099         TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
7100         TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
7101         TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
7102         TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
7103         TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
7104         TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
7105         TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
7106         TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
7107
7108         TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
7109         TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
7110         TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
7111 }
7112
7113 static void tg3_timer(unsigned long __opaque)
7114 {
7115         struct tg3 *tp = (struct tg3 *) __opaque;
7116
7117         if (tp->irq_sync)
7118                 goto restart_timer;
7119
7120         spin_lock(&tp->lock);
7121
7122         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
7123                 /* All of this garbage is because when using non-tagged
7124                  * IRQ status the mailbox/status_block protocol the chip
7125                  * uses with the cpu is race prone.
7126                  */
7127                 if (tp->hw_status->status & SD_STATUS_UPDATED) {
7128                         tw32(GRC_LOCAL_CTRL,
7129                              tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
7130                 } else {
7131                         tw32(HOSTCC_MODE, tp->coalesce_mode |
7132                              (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
7133                 }
7134
7135                 if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
7136                         tp->tg3_flags2 |= TG3_FLG2_RESTART_TIMER;
7137                         spin_unlock(&tp->lock);
7138                         schedule_work(&tp->reset_task);
7139                         return;
7140                 }
7141         }
7142
7143         /* This part only runs once per second. */
7144         if (!--tp->timer_counter) {
7145                 if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
7146                         tg3_periodic_fetch_stats(tp);
7147
7148                 if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
7149                         u32 mac_stat;
7150                         int phy_event;
7151
7152                         mac_stat = tr32(MAC_STATUS);
7153
7154                         phy_event = 0;
7155                         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) {
7156                                 if (mac_stat & MAC_STATUS_MI_INTERRUPT)
7157                                         phy_event = 1;
7158                         } else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
7159                                 phy_event = 1;
7160
7161                         if (phy_event)
7162                                 tg3_setup_phy(tp, 0);
7163                 } else if (tp->tg3_flags & TG3_FLAG_POLL_SERDES) {
7164                         u32 mac_stat = tr32(MAC_STATUS);
7165                         int need_setup = 0;
7166
7167                         if (netif_carrier_ok(tp->dev) &&
7168                             (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
7169                                 need_setup = 1;
7170                         }
7171                         if (! netif_carrier_ok(tp->dev) &&
7172                             (mac_stat & (MAC_STATUS_PCS_SYNCED |
7173                                          MAC_STATUS_SIGNAL_DET))) {
7174                                 need_setup = 1;
7175                         }
7176                         if (need_setup) {
7177                                 if (!tp->serdes_counter) {
7178                                         tw32_f(MAC_MODE,
7179                                              (tp->mac_mode &
7180                                               ~MAC_MODE_PORT_MODE_MASK));
7181                                         udelay(40);
7182                                         tw32_f(MAC_MODE, tp->mac_mode);
7183                                         udelay(40);
7184                                 }
7185                                 tg3_setup_phy(tp, 0);
7186                         }
7187                 } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
7188                         tg3_serdes_parallel_detect(tp);
7189
7190                 tp->timer_counter = tp->timer_multiplier;
7191         }
7192
7193         /* Heartbeat is only sent once every 2 seconds.
7194          *
7195          * The heartbeat is to tell the ASF firmware that the host
7196          * driver is still alive.  In the event that the OS crashes,
7197          * ASF needs to reset the hardware to free up the FIFO space
7198          * that may be filled with rx packets destined for the host.
7199          * If the FIFO is full, ASF will no longer function properly.
7200          *
7201          * Unintended resets have been reported on real time kernels
7202          * where the timer doesn't run on time.  Netpoll will also have
7203          * same problem.
7204          *
7205          * The new FWCMD_NICDRV_ALIVE3 command tells the ASF firmware
7206          * to check the ring condition when the heartbeat is expiring
7207          * before doing the reset.  This will prevent most unintended
7208          * resets.
7209          */
7210         if (!--tp->asf_counter) {
7211                 if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
7212                         u32 val;
7213
7214                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
7215                                       FWCMD_NICDRV_ALIVE3);
7216                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
7217                         /* 5 seconds timeout */
7218                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, 5);
7219                         val = tr32(GRC_RX_CPU_EVENT);
7220                         val |= (1 << 14);
7221                         tw32(GRC_RX_CPU_EVENT, val);
7222                 }
7223                 tp->asf_counter = tp->asf_multiplier;
7224         }
7225
7226         spin_unlock(&tp->lock);
7227
7228 restart_timer:
7229         tp->timer.expires = jiffies + tp->timer_offset;
7230         add_timer(&tp->timer);
7231 }
7232
7233 static int tg3_request_irq(struct tg3 *tp)
7234 {
7235         irq_handler_t fn;
7236         unsigned long flags;
7237         struct net_device *dev = tp->dev;
7238
7239         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7240                 fn = tg3_msi;
7241                 if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
7242                         fn = tg3_msi_1shot;
7243                 flags = IRQF_SAMPLE_RANDOM;
7244         } else {
7245                 fn = tg3_interrupt;
7246                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
7247                         fn = tg3_interrupt_tagged;
7248                 flags = IRQF_SHARED | IRQF_SAMPLE_RANDOM;
7249         }
7250         return (request_irq(tp->pdev->irq, fn, flags, dev->name, dev));
7251 }
7252
7253 static int tg3_test_interrupt(struct tg3 *tp)
7254 {
7255         struct net_device *dev = tp->dev;
7256         int err, i, intr_ok = 0;
7257
7258         if (!netif_running(dev))
7259                 return -ENODEV;
7260
7261         tg3_disable_ints(tp);
7262
7263         free_irq(tp->pdev->irq, dev);
7264
7265         err = request_irq(tp->pdev->irq, tg3_test_isr,
7266                           IRQF_SHARED | IRQF_SAMPLE_RANDOM, dev->name, dev);
7267         if (err)
7268                 return err;
7269
7270         tp->hw_status->status &= ~SD_STATUS_UPDATED;
7271         tg3_enable_ints(tp);
7272
7273         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
7274                HOSTCC_MODE_NOW);
7275
7276         for (i = 0; i < 5; i++) {
7277                 u32 int_mbox, misc_host_ctrl;
7278
7279                 int_mbox = tr32_mailbox(MAILBOX_INTERRUPT_0 +
7280                                         TG3_64BIT_REG_LOW);
7281                 misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
7282
7283                 if ((int_mbox != 0) ||
7284                     (misc_host_ctrl & MISC_HOST_CTRL_MASK_PCI_INT)) {
7285                         intr_ok = 1;
7286                         break;
7287                 }
7288
7289                 msleep(10);
7290         }
7291
7292         tg3_disable_ints(tp);
7293
7294         free_irq(tp->pdev->irq, dev);
7295
7296         err = tg3_request_irq(tp);
7297
7298         if (err)
7299                 return err;
7300
7301         if (intr_ok)
7302                 return 0;
7303
7304         return -EIO;
7305 }
7306
7307 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
7308  * successfully restored
7309  */
7310 static int tg3_test_msi(struct tg3 *tp)
7311 {
7312         struct net_device *dev = tp->dev;
7313         int err;
7314         u16 pci_cmd;
7315
7316         if (!(tp->tg3_flags2 & TG3_FLG2_USING_MSI))
7317                 return 0;
7318
7319         /* Turn off SERR reporting in case MSI terminates with Master
7320          * Abort.
7321          */
7322         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
7323         pci_write_config_word(tp->pdev, PCI_COMMAND,
7324                               pci_cmd & ~PCI_COMMAND_SERR);
7325
7326         err = tg3_test_interrupt(tp);
7327
7328         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
7329
7330         if (!err)
7331                 return 0;
7332
7333         /* other failures */
7334         if (err != -EIO)
7335                 return err;
7336
7337         /* MSI test failed, go back to INTx mode */
7338         printk(KERN_WARNING PFX "%s: No interrupt was generated using MSI, "
7339                "switching to INTx mode. Please report this failure to "
7340                "the PCI maintainer and include system chipset information.\n",
7341                        tp->dev->name);
7342
7343         free_irq(tp->pdev->irq, dev);
7344         pci_disable_msi(tp->pdev);
7345
7346         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7347
7348         err = tg3_request_irq(tp);
7349         if (err)
7350                 return err;
7351
7352         /* Need to reset the chip because the MSI cycle may have terminated
7353          * with Master Abort.
7354          */
7355         tg3_full_lock(tp, 1);
7356
7357         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7358         err = tg3_init_hw(tp, 1);
7359
7360         tg3_full_unlock(tp);
7361
7362         if (err)
7363                 free_irq(tp->pdev->irq, dev);
7364
7365         return err;
7366 }
7367
7368 static int tg3_open(struct net_device *dev)
7369 {
7370         struct tg3 *tp = netdev_priv(dev);
7371         int err;
7372
7373         netif_carrier_off(tp->dev);
7374
7375         tg3_full_lock(tp, 0);
7376
7377         err = tg3_set_power_state(tp, PCI_D0);
7378         if (err) {
7379                 tg3_full_unlock(tp);
7380                 return err;
7381         }
7382
7383         tg3_disable_ints(tp);
7384         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
7385
7386         tg3_full_unlock(tp);
7387
7388         /* The placement of this call is tied
7389          * to the setup and use of Host TX descriptors.
7390          */
7391         err = tg3_alloc_consistent(tp);
7392         if (err)
7393                 return err;
7394
7395         if (tp->tg3_flags & TG3_FLAG_SUPPORT_MSI) {
7396                 /* All MSI supporting chips should support tagged
7397                  * status.  Assert that this is the case.
7398                  */
7399                 if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
7400                         printk(KERN_WARNING PFX "%s: MSI without TAGGED? "
7401                                "Not using MSI.\n", tp->dev->name);
7402                 } else if (pci_enable_msi(tp->pdev) == 0) {
7403                         u32 msi_mode;
7404
7405                         msi_mode = tr32(MSGINT_MODE);
7406                         tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
7407                         tp->tg3_flags2 |= TG3_FLG2_USING_MSI;
7408                 }
7409         }
7410         err = tg3_request_irq(tp);
7411
7412         if (err) {
7413                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7414                         pci_disable_msi(tp->pdev);
7415                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7416                 }
7417                 tg3_free_consistent(tp);
7418                 return err;
7419         }
7420
7421         napi_enable(&tp->napi);
7422
7423         tg3_full_lock(tp, 0);
7424
7425         err = tg3_init_hw(tp, 1);
7426         if (err) {
7427                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7428                 tg3_free_rings(tp);
7429         } else {
7430                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
7431                         tp->timer_offset = HZ;
7432                 else
7433                         tp->timer_offset = HZ / 10;
7434
7435                 BUG_ON(tp->timer_offset > HZ);
7436                 tp->timer_counter = tp->timer_multiplier =
7437                         (HZ / tp->timer_offset);
7438                 tp->asf_counter = tp->asf_multiplier =
7439                         ((HZ / tp->timer_offset) * 2);
7440
7441                 init_timer(&tp->timer);
7442                 tp->timer.expires = jiffies + tp->timer_offset;
7443                 tp->timer.data = (unsigned long) tp;
7444                 tp->timer.function = tg3_timer;
7445         }
7446
7447         tg3_full_unlock(tp);
7448
7449         if (err) {
7450                 napi_disable(&tp->napi);
7451                 free_irq(tp->pdev->irq, dev);
7452                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7453                         pci_disable_msi(tp->pdev);
7454                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7455                 }
7456                 tg3_free_consistent(tp);
7457                 return err;
7458         }
7459
7460         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7461                 err = tg3_test_msi(tp);
7462
7463                 if (err) {
7464                         tg3_full_lock(tp, 0);
7465
7466                         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7467                                 pci_disable_msi(tp->pdev);
7468                                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7469                         }
7470                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7471                         tg3_free_rings(tp);
7472                         tg3_free_consistent(tp);
7473
7474                         tg3_full_unlock(tp);
7475
7476                         napi_disable(&tp->napi);
7477
7478                         return err;
7479                 }
7480
7481                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7482                         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI) {
7483                                 u32 val = tr32(PCIE_TRANSACTION_CFG);
7484
7485                                 tw32(PCIE_TRANSACTION_CFG,
7486                                      val | PCIE_TRANS_CFG_1SHOT_MSI);
7487                         }
7488                 }
7489         }
7490
7491         tg3_full_lock(tp, 0);
7492
7493         add_timer(&tp->timer);
7494         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
7495         tg3_enable_ints(tp);
7496
7497         tg3_full_unlock(tp);
7498
7499         netif_start_queue(dev);
7500
7501         return 0;
7502 }
7503
7504 #if 0
7505 /*static*/ void tg3_dump_state(struct tg3 *tp)
7506 {
7507         u32 val32, val32_2, val32_3, val32_4, val32_5;
7508         u16 val16;
7509         int i;
7510
7511         pci_read_config_word(tp->pdev, PCI_STATUS, &val16);
7512         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE, &val32);
7513         printk("DEBUG: PCI status [%04x] TG3PCI state[%08x]\n",
7514                val16, val32);
7515
7516         /* MAC block */
7517         printk("DEBUG: MAC_MODE[%08x] MAC_STATUS[%08x]\n",
7518                tr32(MAC_MODE), tr32(MAC_STATUS));
7519         printk("       MAC_EVENT[%08x] MAC_LED_CTRL[%08x]\n",
7520                tr32(MAC_EVENT), tr32(MAC_LED_CTRL));
7521         printk("DEBUG: MAC_TX_MODE[%08x] MAC_TX_STATUS[%08x]\n",
7522                tr32(MAC_TX_MODE), tr32(MAC_TX_STATUS));
7523         printk("       MAC_RX_MODE[%08x] MAC_RX_STATUS[%08x]\n",
7524                tr32(MAC_RX_MODE), tr32(MAC_RX_STATUS));
7525
7526         /* Send data initiator control block */
7527         printk("DEBUG: SNDDATAI_MODE[%08x] SNDDATAI_STATUS[%08x]\n",
7528                tr32(SNDDATAI_MODE), tr32(SNDDATAI_STATUS));
7529         printk("       SNDDATAI_STATSCTRL[%08x]\n",
7530                tr32(SNDDATAI_STATSCTRL));
7531
7532         /* Send data completion control block */
7533         printk("DEBUG: SNDDATAC_MODE[%08x]\n", tr32(SNDDATAC_MODE));
7534
7535         /* Send BD ring selector block */
7536         printk("DEBUG: SNDBDS_MODE[%08x] SNDBDS_STATUS[%08x]\n",
7537                tr32(SNDBDS_MODE), tr32(SNDBDS_STATUS));
7538
7539         /* Send BD initiator control block */
7540         printk("DEBUG: SNDBDI_MODE[%08x] SNDBDI_STATUS[%08x]\n",
7541                tr32(SNDBDI_MODE), tr32(SNDBDI_STATUS));
7542
7543         /* Send BD completion control block */
7544         printk("DEBUG: SNDBDC_MODE[%08x]\n", tr32(SNDBDC_MODE));
7545
7546         /* Receive list placement control block */
7547         printk("DEBUG: RCVLPC_MODE[%08x] RCVLPC_STATUS[%08x]\n",
7548                tr32(RCVLPC_MODE), tr32(RCVLPC_STATUS));
7549         printk("       RCVLPC_STATSCTRL[%08x]\n",
7550                tr32(RCVLPC_STATSCTRL));
7551
7552         /* Receive data and receive BD initiator control block */
7553         printk("DEBUG: RCVDBDI_MODE[%08x] RCVDBDI_STATUS[%08x]\n",
7554                tr32(RCVDBDI_MODE), tr32(RCVDBDI_STATUS));
7555
7556         /* Receive data completion control block */
7557         printk("DEBUG: RCVDCC_MODE[%08x]\n",
7558                tr32(RCVDCC_MODE));
7559
7560         /* Receive BD initiator control block */
7561         printk("DEBUG: RCVBDI_MODE[%08x] RCVBDI_STATUS[%08x]\n",
7562                tr32(RCVBDI_MODE), tr32(RCVBDI_STATUS));
7563
7564         /* Receive BD completion control block */
7565         printk("DEBUG: RCVCC_MODE[%08x] RCVCC_STATUS[%08x]\n",
7566                tr32(RCVCC_MODE), tr32(RCVCC_STATUS));
7567
7568         /* Receive list selector control block */
7569         printk("DEBUG: RCVLSC_MODE[%08x] RCVLSC_STATUS[%08x]\n",
7570                tr32(RCVLSC_MODE), tr32(RCVLSC_STATUS));
7571
7572         /* Mbuf cluster free block */
7573         printk("DEBUG: MBFREE_MODE[%08x] MBFREE_STATUS[%08x]\n",
7574                tr32(MBFREE_MODE), tr32(MBFREE_STATUS));
7575
7576         /* Host coalescing control block */
7577         printk("DEBUG: HOSTCC_MODE[%08x] HOSTCC_STATUS[%08x]\n",
7578                tr32(HOSTCC_MODE), tr32(HOSTCC_STATUS));
7579         printk("DEBUG: HOSTCC_STATS_BLK_HOST_ADDR[%08x%08x]\n",
7580                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
7581                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
7582         printk("DEBUG: HOSTCC_STATUS_BLK_HOST_ADDR[%08x%08x]\n",
7583                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
7584                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
7585         printk("DEBUG: HOSTCC_STATS_BLK_NIC_ADDR[%08x]\n",
7586                tr32(HOSTCC_STATS_BLK_NIC_ADDR));
7587         printk("DEBUG: HOSTCC_STATUS_BLK_NIC_ADDR[%08x]\n",
7588                tr32(HOSTCC_STATUS_BLK_NIC_ADDR));
7589
7590         /* Memory arbiter control block */
7591         printk("DEBUG: MEMARB_MODE[%08x] MEMARB_STATUS[%08x]\n",
7592                tr32(MEMARB_MODE), tr32(MEMARB_STATUS));
7593
7594         /* Buffer manager control block */
7595         printk("DEBUG: BUFMGR_MODE[%08x] BUFMGR_STATUS[%08x]\n",
7596                tr32(BUFMGR_MODE), tr32(BUFMGR_STATUS));
7597         printk("DEBUG: BUFMGR_MB_POOL_ADDR[%08x] BUFMGR_MB_POOL_SIZE[%08x]\n",
7598                tr32(BUFMGR_MB_POOL_ADDR), tr32(BUFMGR_MB_POOL_SIZE));
7599         printk("DEBUG: BUFMGR_DMA_DESC_POOL_ADDR[%08x] "
7600                "BUFMGR_DMA_DESC_POOL_SIZE[%08x]\n",
7601                tr32(BUFMGR_DMA_DESC_POOL_ADDR),
7602                tr32(BUFMGR_DMA_DESC_POOL_SIZE));
7603
7604         /* Read DMA control block */
7605         printk("DEBUG: RDMAC_MODE[%08x] RDMAC_STATUS[%08x]\n",
7606                tr32(RDMAC_MODE), tr32(RDMAC_STATUS));
7607
7608         /* Write DMA control block */
7609         printk("DEBUG: WDMAC_MODE[%08x] WDMAC_STATUS[%08x]\n",
7610                tr32(WDMAC_MODE), tr32(WDMAC_STATUS));
7611
7612         /* DMA completion block */
7613         printk("DEBUG: DMAC_MODE[%08x]\n",
7614                tr32(DMAC_MODE));
7615
7616         /* GRC block */
7617         printk("DEBUG: GRC_MODE[%08x] GRC_MISC_CFG[%08x]\n",
7618                tr32(GRC_MODE), tr32(GRC_MISC_CFG));
7619         printk("DEBUG: GRC_LOCAL_CTRL[%08x]\n",
7620                tr32(GRC_LOCAL_CTRL));
7621
7622         /* TG3_BDINFOs */
7623         printk("DEBUG: RCVDBDI_JUMBO_BD[%08x%08x:%08x:%08x]\n",
7624                tr32(RCVDBDI_JUMBO_BD + 0x0),
7625                tr32(RCVDBDI_JUMBO_BD + 0x4),
7626                tr32(RCVDBDI_JUMBO_BD + 0x8),
7627                tr32(RCVDBDI_JUMBO_BD + 0xc));
7628         printk("DEBUG: RCVDBDI_STD_BD[%08x%08x:%08x:%08x]\n",
7629                tr32(RCVDBDI_STD_BD + 0x0),
7630                tr32(RCVDBDI_STD_BD + 0x4),
7631                tr32(RCVDBDI_STD_BD + 0x8),
7632                tr32(RCVDBDI_STD_BD + 0xc));
7633         printk("DEBUG: RCVDBDI_MINI_BD[%08x%08x:%08x:%08x]\n",
7634                tr32(RCVDBDI_MINI_BD + 0x0),
7635                tr32(RCVDBDI_MINI_BD + 0x4),
7636                tr32(RCVDBDI_MINI_BD + 0x8),
7637                tr32(RCVDBDI_MINI_BD + 0xc));
7638
7639         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x0, &val32);
7640         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x4, &val32_2);
7641         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x8, &val32_3);
7642         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0xc, &val32_4);
7643         printk("DEBUG: SRAM_SEND_RCB_0[%08x%08x:%08x:%08x]\n",
7644                val32, val32_2, val32_3, val32_4);
7645
7646         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x0, &val32);
7647         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x4, &val32_2);
7648         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x8, &val32_3);
7649         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0xc, &val32_4);
7650         printk("DEBUG: SRAM_RCV_RET_RCB_0[%08x%08x:%08x:%08x]\n",
7651                val32, val32_2, val32_3, val32_4);
7652
7653         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x0, &val32);
7654         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x4, &val32_2);
7655         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x8, &val32_3);
7656         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0xc, &val32_4);
7657         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x10, &val32_5);
7658         printk("DEBUG: SRAM_STATUS_BLK[%08x:%08x:%08x:%08x:%08x]\n",
7659                val32, val32_2, val32_3, val32_4, val32_5);
7660
7661         /* SW status block */
7662         printk("DEBUG: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
7663                tp->hw_status->status,
7664                tp->hw_status->status_tag,
7665                tp->hw_status->rx_jumbo_consumer,
7666                tp->hw_status->rx_consumer,
7667                tp->hw_status->rx_mini_consumer,
7668                tp->hw_status->idx[0].rx_producer,
7669                tp->hw_status->idx[0].tx_consumer);
7670
7671         /* SW statistics block */
7672         printk("DEBUG: Host statistics block [%08x:%08x:%08x:%08x]\n",
7673                ((u32 *)tp->hw_stats)[0],
7674                ((u32 *)tp->hw_stats)[1],
7675                ((u32 *)tp->hw_stats)[2],
7676                ((u32 *)tp->hw_stats)[3]);
7677
7678         /* Mailboxes */
7679         printk("DEBUG: SNDHOST_PROD[%08x%08x] SNDNIC_PROD[%08x%08x]\n",
7680                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x0),
7681                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x4),
7682                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x0),
7683                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x4));
7684
7685         /* NIC side send descriptors. */
7686         for (i = 0; i < 6; i++) {
7687                 unsigned long txd;
7688
7689                 txd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_TX_BUFFER_DESC
7690                         + (i * sizeof(struct tg3_tx_buffer_desc));
7691                 printk("DEBUG: NIC TXD(%d)[%08x:%08x:%08x:%08x]\n",
7692                        i,
7693                        readl(txd + 0x0), readl(txd + 0x4),
7694                        readl(txd + 0x8), readl(txd + 0xc));
7695         }
7696
7697         /* NIC side RX descriptors. */
7698         for (i = 0; i < 6; i++) {
7699                 unsigned long rxd;
7700
7701                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_BUFFER_DESC
7702                         + (i * sizeof(struct tg3_rx_buffer_desc));
7703                 printk("DEBUG: NIC RXD_STD(%d)[0][%08x:%08x:%08x:%08x]\n",
7704                        i,
7705                        readl(rxd + 0x0), readl(rxd + 0x4),
7706                        readl(rxd + 0x8), readl(rxd + 0xc));
7707                 rxd += (4 * sizeof(u32));
7708                 printk("DEBUG: NIC RXD_STD(%d)[1][%08x:%08x:%08x:%08x]\n",
7709                        i,
7710                        readl(rxd + 0x0), readl(rxd + 0x4),
7711                        readl(rxd + 0x8), readl(rxd + 0xc));
7712         }
7713
7714         for (i = 0; i < 6; i++) {
7715                 unsigned long rxd;
7716
7717                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_JUMBO_BUFFER_DESC
7718                         + (i * sizeof(struct tg3_rx_buffer_desc));
7719                 printk("DEBUG: NIC RXD_JUMBO(%d)[0][%08x:%08x:%08x:%08x]\n",
7720                        i,
7721                        readl(rxd + 0x0), readl(rxd + 0x4),
7722                        readl(rxd + 0x8), readl(rxd + 0xc));
7723                 rxd += (4 * sizeof(u32));
7724                 printk("DEBUG: NIC RXD_JUMBO(%d)[1][%08x:%08x:%08x:%08x]\n",
7725                        i,
7726                        readl(rxd + 0x0), readl(rxd + 0x4),
7727                        readl(rxd + 0x8), readl(rxd + 0xc));
7728         }
7729 }
7730 #endif
7731
7732 static struct net_device_stats *tg3_get_stats(struct net_device *);
7733 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *);
7734
7735 static int tg3_close(struct net_device *dev)
7736 {
7737         struct tg3 *tp = netdev_priv(dev);
7738
7739         napi_disable(&tp->napi);
7740         cancel_work_sync(&tp->reset_task);
7741
7742         netif_stop_queue(dev);
7743
7744         del_timer_sync(&tp->timer);
7745
7746         tg3_full_lock(tp, 1);
7747 #if 0
7748         tg3_dump_state(tp);
7749 #endif
7750
7751         tg3_disable_ints(tp);
7752
7753         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7754         tg3_free_rings(tp);
7755         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
7756
7757         tg3_full_unlock(tp);
7758
7759         free_irq(tp->pdev->irq, dev);
7760         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7761                 pci_disable_msi(tp->pdev);
7762                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7763         }
7764
7765         memcpy(&tp->net_stats_prev, tg3_get_stats(tp->dev),
7766                sizeof(tp->net_stats_prev));
7767         memcpy(&tp->estats_prev, tg3_get_estats(tp),
7768                sizeof(tp->estats_prev));
7769
7770         tg3_free_consistent(tp);
7771
7772         tg3_set_power_state(tp, PCI_D3hot);
7773
7774         netif_carrier_off(tp->dev);
7775
7776         return 0;
7777 }
7778
7779 static inline unsigned long get_stat64(tg3_stat64_t *val)
7780 {
7781         unsigned long ret;
7782
7783 #if (BITS_PER_LONG == 32)
7784         ret = val->low;
7785 #else
7786         ret = ((u64)val->high << 32) | ((u64)val->low);
7787 #endif
7788         return ret;
7789 }
7790
7791 static unsigned long calc_crc_errors(struct tg3 *tp)
7792 {
7793         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7794
7795         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7796             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
7797              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
7798                 u32 val;
7799
7800                 spin_lock_bh(&tp->lock);
7801                 if (!tg3_readphy(tp, MII_TG3_TEST1, &val)) {
7802                         tg3_writephy(tp, MII_TG3_TEST1,
7803                                      val | MII_TG3_TEST1_CRC_EN);
7804                         tg3_readphy(tp, 0x14, &val);
7805                 } else
7806                         val = 0;
7807                 spin_unlock_bh(&tp->lock);
7808
7809                 tp->phy_crc_errors += val;
7810
7811                 return tp->phy_crc_errors;
7812         }
7813
7814         return get_stat64(&hw_stats->rx_fcs_errors);
7815 }
7816
7817 #define ESTAT_ADD(member) \
7818         estats->member =        old_estats->member + \
7819                                 get_stat64(&hw_stats->member)
7820
7821 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *tp)
7822 {
7823         struct tg3_ethtool_stats *estats = &tp->estats;
7824         struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
7825         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7826
7827         if (!hw_stats)
7828                 return old_estats;
7829
7830         ESTAT_ADD(rx_octets);
7831         ESTAT_ADD(rx_fragments);
7832         ESTAT_ADD(rx_ucast_packets);
7833         ESTAT_ADD(rx_mcast_packets);
7834         ESTAT_ADD(rx_bcast_packets);
7835         ESTAT_ADD(rx_fcs_errors);
7836         ESTAT_ADD(rx_align_errors);
7837         ESTAT_ADD(rx_xon_pause_rcvd);
7838         ESTAT_ADD(rx_xoff_pause_rcvd);
7839         ESTAT_ADD(rx_mac_ctrl_rcvd);
7840         ESTAT_ADD(rx_xoff_entered);
7841         ESTAT_ADD(rx_frame_too_long_errors);
7842         ESTAT_ADD(rx_jabbers);
7843         ESTAT_ADD(rx_undersize_packets);
7844         ESTAT_ADD(rx_in_length_errors);
7845         ESTAT_ADD(rx_out_length_errors);
7846         ESTAT_ADD(rx_64_or_less_octet_packets);
7847         ESTAT_ADD(rx_65_to_127_octet_packets);
7848         ESTAT_ADD(rx_128_to_255_octet_packets);
7849         ESTAT_ADD(rx_256_to_511_octet_packets);
7850         ESTAT_ADD(rx_512_to_1023_octet_packets);
7851         ESTAT_ADD(rx_1024_to_1522_octet_packets);
7852         ESTAT_ADD(rx_1523_to_2047_octet_packets);
7853         ESTAT_ADD(rx_2048_to_4095_octet_packets);
7854         ESTAT_ADD(rx_4096_to_8191_octet_packets);
7855         ESTAT_ADD(rx_8192_to_9022_octet_packets);
7856
7857         ESTAT_ADD(tx_octets);
7858         ESTAT_ADD(tx_collisions);
7859         ESTAT_ADD(tx_xon_sent);
7860         ESTAT_ADD(tx_xoff_sent);
7861         ESTAT_ADD(tx_flow_control);
7862         ESTAT_ADD(tx_mac_errors);
7863         ESTAT_ADD(tx_single_collisions);
7864         ESTAT_ADD(tx_mult_collisions);
7865         ESTAT_ADD(tx_deferred);
7866         ESTAT_ADD(tx_excessive_collisions);
7867         ESTAT_ADD(tx_late_collisions);
7868         ESTAT_ADD(tx_collide_2times);
7869         ESTAT_ADD(tx_collide_3times);
7870         ESTAT_ADD(tx_collide_4times);
7871         ESTAT_ADD(tx_collide_5times);
7872         ESTAT_ADD(tx_collide_6times);
7873         ESTAT_ADD(tx_collide_7times);
7874         ESTAT_ADD(tx_collide_8times);
7875         ESTAT_ADD(tx_collide_9times);
7876         ESTAT_ADD(tx_collide_10times);
7877         ESTAT_ADD(tx_collide_11times);
7878         ESTAT_ADD(tx_collide_12times);
7879         ESTAT_ADD(tx_collide_13times);
7880         ESTAT_ADD(tx_collide_14times);
7881         ESTAT_ADD(tx_collide_15times);
7882         ESTAT_ADD(tx_ucast_packets);
7883         ESTAT_ADD(tx_mcast_packets);
7884         ESTAT_ADD(tx_bcast_packets);
7885         ESTAT_ADD(tx_carrier_sense_errors);
7886         ESTAT_ADD(tx_discards);
7887         ESTAT_ADD(tx_errors);
7888
7889         ESTAT_ADD(dma_writeq_full);
7890         ESTAT_ADD(dma_write_prioq_full);
7891         ESTAT_ADD(rxbds_empty);
7892         ESTAT_ADD(rx_discards);
7893         ESTAT_ADD(rx_errors);
7894         ESTAT_ADD(rx_threshold_hit);
7895
7896         ESTAT_ADD(dma_readq_full);
7897         ESTAT_ADD(dma_read_prioq_full);
7898         ESTAT_ADD(tx_comp_queue_full);
7899
7900         ESTAT_ADD(ring_set_send_prod_index);
7901         ESTAT_ADD(ring_status_update);
7902         ESTAT_ADD(nic_irqs);
7903         ESTAT_ADD(nic_avoided_irqs);
7904         ESTAT_ADD(nic_tx_threshold_hit);
7905
7906         return estats;
7907 }
7908
7909 static struct net_device_stats *tg3_get_stats(struct net_device *dev)
7910 {
7911         struct tg3 *tp = netdev_priv(dev);
7912         struct net_device_stats *stats = &tp->net_stats;
7913         struct net_device_stats *old_stats = &tp->net_stats_prev;
7914         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7915
7916         if (!hw_stats)
7917                 return old_stats;
7918
7919         stats->rx_packets = old_stats->rx_packets +
7920                 get_stat64(&hw_stats->rx_ucast_packets) +
7921                 get_stat64(&hw_stats->rx_mcast_packets) +
7922                 get_stat64(&hw_stats->rx_bcast_packets);
7923
7924         stats->tx_packets = old_stats->tx_packets +
7925                 get_stat64(&hw_stats->tx_ucast_packets) +
7926                 get_stat64(&hw_stats->tx_mcast_packets) +
7927                 get_stat64(&hw_stats->tx_bcast_packets);
7928
7929         stats->rx_bytes = old_stats->rx_bytes +
7930                 get_stat64(&hw_stats->rx_octets);
7931         stats->tx_bytes = old_stats->tx_bytes +
7932                 get_stat64(&hw_stats->tx_octets);
7933
7934         stats->rx_errors = old_stats->rx_errors +
7935                 get_stat64(&hw_stats->rx_errors);
7936         stats->tx_errors = old_stats->tx_errors +
7937                 get_stat64(&hw_stats->tx_errors) +
7938                 get_stat64(&hw_stats->tx_mac_errors) +
7939                 get_stat64(&hw_stats->tx_carrier_sense_errors) +
7940                 get_stat64(&hw_stats->tx_discards);
7941
7942         stats->multicast = old_stats->multicast +
7943                 get_stat64(&hw_stats->rx_mcast_packets);
7944         stats->collisions = old_stats->collisions +
7945                 get_stat64(&hw_stats->tx_collisions);
7946
7947         stats->rx_length_errors = old_stats->rx_length_errors +
7948                 get_stat64(&hw_stats->rx_frame_too_long_errors) +
7949                 get_stat64(&hw_stats->rx_undersize_packets);
7950
7951         stats->rx_over_errors = old_stats->rx_over_errors +
7952                 get_stat64(&hw_stats->rxbds_empty);
7953         stats->rx_frame_errors = old_stats->rx_frame_errors +
7954                 get_stat64(&hw_stats->rx_align_errors);
7955         stats->tx_aborted_errors = old_stats->tx_aborted_errors +
7956                 get_stat64(&hw_stats->tx_discards);
7957         stats->tx_carrier_errors = old_stats->tx_carrier_errors +
7958                 get_stat64(&hw_stats->tx_carrier_sense_errors);
7959
7960         stats->rx_crc_errors = old_stats->rx_crc_errors +
7961                 calc_crc_errors(tp);
7962
7963         stats->rx_missed_errors = old_stats->rx_missed_errors +
7964                 get_stat64(&hw_stats->rx_discards);
7965
7966         return stats;
7967 }
7968
7969 static inline u32 calc_crc(unsigned char *buf, int len)
7970 {
7971         u32 reg;
7972         u32 tmp;
7973         int j, k;
7974
7975         reg = 0xffffffff;
7976
7977         for (j = 0; j < len; j++) {
7978                 reg ^= buf[j];
7979
7980                 for (k = 0; k < 8; k++) {
7981                         tmp = reg & 0x01;
7982
7983                         reg >>= 1;
7984
7985                         if (tmp) {
7986                                 reg ^= 0xedb88320;
7987                         }
7988                 }
7989         }
7990
7991         return ~reg;
7992 }
7993
7994 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
7995 {
7996         /* accept or reject all multicast frames */
7997         tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
7998         tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
7999         tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
8000         tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
8001 }
8002
8003 static void __tg3_set_rx_mode(struct net_device *dev)
8004 {
8005         struct tg3 *tp = netdev_priv(dev);
8006         u32 rx_mode;
8007
8008         rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
8009                                   RX_MODE_KEEP_VLAN_TAG);
8010
8011         /* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
8012          * flag clear.
8013          */
8014 #if TG3_VLAN_TAG_USED
8015         if (!tp->vlgrp &&
8016             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
8017                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
8018 #else
8019         /* By definition, VLAN is disabled always in this
8020          * case.
8021          */
8022         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
8023                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
8024 #endif
8025
8026         if (dev->flags & IFF_PROMISC) {
8027                 /* Promiscuous mode. */
8028                 rx_mode |= RX_MODE_PROMISC;
8029         } else if (dev->flags & IFF_ALLMULTI) {
8030                 /* Accept all multicast. */
8031                 tg3_set_multi (tp, 1);
8032         } else if (dev->mc_count < 1) {
8033                 /* Reject all multicast. */
8034                 tg3_set_multi (tp, 0);
8035         } else {
8036                 /* Accept one or more multicast(s). */
8037                 struct dev_mc_list *mclist;
8038                 unsigned int i;
8039                 u32 mc_filter[4] = { 0, };
8040                 u32 regidx;
8041                 u32 bit;
8042                 u32 crc;
8043
8044                 for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count;
8045                      i++, mclist = mclist->next) {
8046
8047                         crc = calc_crc (mclist->dmi_addr, ETH_ALEN);
8048                         bit = ~crc & 0x7f;
8049                         regidx = (bit & 0x60) >> 5;
8050                         bit &= 0x1f;
8051                         mc_filter[regidx] |= (1 << bit);
8052                 }
8053
8054                 tw32(MAC_HASH_REG_0, mc_filter[0]);
8055                 tw32(MAC_HASH_REG_1, mc_filter[1]);
8056                 tw32(MAC_HASH_REG_2, mc_filter[2]);
8057                 tw32(MAC_HASH_REG_3, mc_filter[3]);
8058         }
8059
8060         if (rx_mode != tp->rx_mode) {
8061                 tp->rx_mode = rx_mode;
8062                 tw32_f(MAC_RX_MODE, rx_mode);
8063                 udelay(10);
8064         }
8065 }
8066
8067 static void tg3_set_rx_mode(struct net_device *dev)
8068 {
8069         struct tg3 *tp = netdev_priv(dev);
8070
8071         if (!netif_running(dev))
8072                 return;
8073
8074         tg3_full_lock(tp, 0);
8075         __tg3_set_rx_mode(dev);
8076         tg3_full_unlock(tp);
8077 }
8078
8079 #define TG3_REGDUMP_LEN         (32 * 1024)
8080
8081 static int tg3_get_regs_len(struct net_device *dev)
8082 {
8083         return TG3_REGDUMP_LEN;
8084 }
8085
8086 static void tg3_get_regs(struct net_device *dev,
8087                 struct ethtool_regs *regs, void *_p)
8088 {
8089         u32 *p = _p;
8090         struct tg3 *tp = netdev_priv(dev);
8091         u8 *orig_p = _p;
8092         int i;
8093
8094         regs->version = 0;
8095
8096         memset(p, 0, TG3_REGDUMP_LEN);
8097
8098         if (tp->link_config.phy_is_low_power)
8099                 return;
8100
8101         tg3_full_lock(tp, 0);
8102
8103 #define __GET_REG32(reg)        (*(p)++ = tr32(reg))
8104 #define GET_REG32_LOOP(base,len)                \
8105 do {    p = (u32 *)(orig_p + (base));           \
8106         for (i = 0; i < len; i += 4)            \
8107                 __GET_REG32((base) + i);        \
8108 } while (0)
8109 #define GET_REG32_1(reg)                        \
8110 do {    p = (u32 *)(orig_p + (reg));            \
8111         __GET_REG32((reg));                     \
8112 } while (0)
8113
8114         GET_REG32_LOOP(TG3PCI_VENDOR, 0xb0);
8115         GET_REG32_LOOP(MAILBOX_INTERRUPT_0, 0x200);
8116         GET_REG32_LOOP(MAC_MODE, 0x4f0);
8117         GET_REG32_LOOP(SNDDATAI_MODE, 0xe0);
8118         GET_REG32_1(SNDDATAC_MODE);
8119         GET_REG32_LOOP(SNDBDS_MODE, 0x80);
8120         GET_REG32_LOOP(SNDBDI_MODE, 0x48);
8121         GET_REG32_1(SNDBDC_MODE);
8122         GET_REG32_LOOP(RCVLPC_MODE, 0x20);
8123         GET_REG32_LOOP(RCVLPC_SELLST_BASE, 0x15c);
8124         GET_REG32_LOOP(RCVDBDI_MODE, 0x0c);
8125         GET_REG32_LOOP(RCVDBDI_JUMBO_BD, 0x3c);
8126         GET_REG32_LOOP(RCVDBDI_BD_PROD_IDX_0, 0x44);
8127         GET_REG32_1(RCVDCC_MODE);
8128         GET_REG32_LOOP(RCVBDI_MODE, 0x20);
8129         GET_REG32_LOOP(RCVCC_MODE, 0x14);
8130         GET_REG32_LOOP(RCVLSC_MODE, 0x08);
8131         GET_REG32_1(MBFREE_MODE);
8132         GET_REG32_LOOP(HOSTCC_MODE, 0x100);
8133         GET_REG32_LOOP(MEMARB_MODE, 0x10);
8134         GET_REG32_LOOP(BUFMGR_MODE, 0x58);
8135         GET_REG32_LOOP(RDMAC_MODE, 0x08);
8136         GET_REG32_LOOP(WDMAC_MODE, 0x08);
8137         GET_REG32_1(RX_CPU_MODE);
8138         GET_REG32_1(RX_CPU_STATE);
8139         GET_REG32_1(RX_CPU_PGMCTR);
8140         GET_REG32_1(RX_CPU_HWBKPT);
8141         GET_REG32_1(TX_CPU_MODE);
8142         GET_REG32_1(TX_CPU_STATE);
8143         GET_REG32_1(TX_CPU_PGMCTR);
8144         GET_REG32_LOOP(GRCMBOX_INTERRUPT_0, 0x110);
8145         GET_REG32_LOOP(FTQ_RESET, 0x120);
8146         GET_REG32_LOOP(MSGINT_MODE, 0x0c);
8147         GET_REG32_1(DMAC_MODE);
8148         GET_REG32_LOOP(GRC_MODE, 0x4c);
8149         if (tp->tg3_flags & TG3_FLAG_NVRAM)
8150                 GET_REG32_LOOP(NVRAM_CMD, 0x24);
8151
8152 #undef __GET_REG32
8153 #undef GET_REG32_LOOP
8154 #undef GET_REG32_1
8155
8156         tg3_full_unlock(tp);
8157 }
8158
8159 static int tg3_get_eeprom_len(struct net_device *dev)
8160 {
8161         struct tg3 *tp = netdev_priv(dev);
8162
8163         return tp->nvram_size;
8164 }
8165
8166 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val);
8167 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val);
8168
8169 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
8170 {
8171         struct tg3 *tp = netdev_priv(dev);
8172         int ret;
8173         u8  *pd;
8174         u32 i, offset, len, val, b_offset, b_count;
8175
8176         if (tp->link_config.phy_is_low_power)
8177                 return -EAGAIN;
8178
8179         offset = eeprom->offset;
8180         len = eeprom->len;
8181         eeprom->len = 0;
8182
8183         eeprom->magic = TG3_EEPROM_MAGIC;
8184
8185         if (offset & 3) {
8186                 /* adjustments to start on required 4 byte boundary */
8187                 b_offset = offset & 3;
8188                 b_count = 4 - b_offset;
8189                 if (b_count > len) {
8190                         /* i.e. offset=1 len=2 */
8191                         b_count = len;
8192                 }
8193                 ret = tg3_nvram_read(tp, offset-b_offset, &val);
8194                 if (ret)
8195                         return ret;
8196                 val = cpu_to_le32(val);
8197                 memcpy(data, ((char*)&val) + b_offset, b_count);
8198                 len -= b_count;
8199                 offset += b_count;
8200                 eeprom->len += b_count;
8201         }
8202
8203         /* read bytes upto the last 4 byte boundary */
8204         pd = &data[eeprom->len];
8205         for (i = 0; i < (len - (len & 3)); i += 4) {
8206                 ret = tg3_nvram_read(tp, offset + i, &val);
8207                 if (ret) {
8208                         eeprom->len += i;
8209                         return ret;
8210                 }
8211                 val = cpu_to_le32(val);
8212                 memcpy(pd + i, &val, 4);
8213         }
8214         eeprom->len += i;
8215
8216         if (len & 3) {
8217                 /* read last bytes not ending on 4 byte boundary */
8218                 pd = &data[eeprom->len];
8219                 b_count = len & 3;
8220                 b_offset = offset + len - b_count;
8221                 ret = tg3_nvram_read(tp, b_offset, &val);
8222                 if (ret)
8223                         return ret;
8224                 val = cpu_to_le32(val);
8225                 memcpy(pd, ((char*)&val), b_count);
8226                 eeprom->len += b_count;
8227         }
8228         return 0;
8229 }
8230
8231 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf);
8232
8233 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
8234 {
8235         struct tg3 *tp = netdev_priv(dev);
8236         int ret;
8237         u32 offset, len, b_offset, odd_len, start, end;
8238         u8 *buf;
8239
8240         if (tp->link_config.phy_is_low_power)
8241                 return -EAGAIN;
8242
8243         if (eeprom->magic != TG3_EEPROM_MAGIC)
8244                 return -EINVAL;
8245
8246         offset = eeprom->offset;
8247         len = eeprom->len;
8248
8249         if ((b_offset = (offset & 3))) {
8250                 /* adjustments to start on required 4 byte boundary */
8251                 ret = tg3_nvram_read(tp, offset-b_offset, &start);
8252                 if (ret)
8253                         return ret;
8254                 start = cpu_to_le32(start);
8255                 len += b_offset;
8256                 offset &= ~3;
8257                 if (len < 4)
8258                         len = 4;
8259         }
8260
8261         odd_len = 0;
8262         if (len & 3) {
8263                 /* adjustments to end on required 4 byte boundary */
8264                 odd_len = 1;
8265                 len = (len + 3) & ~3;
8266                 ret = tg3_nvram_read(tp, offset+len-4, &end);
8267                 if (ret)
8268                         return ret;
8269                 end = cpu_to_le32(end);
8270         }
8271
8272         buf = data;
8273         if (b_offset || odd_len) {
8274                 buf = kmalloc(len, GFP_KERNEL);
8275                 if (!buf)
8276                         return -ENOMEM;
8277                 if (b_offset)
8278                         memcpy(buf, &start, 4);
8279                 if (odd_len)
8280                         memcpy(buf+len-4, &end, 4);
8281                 memcpy(buf + b_offset, data, eeprom->len);
8282         }
8283
8284         ret = tg3_nvram_write_block(tp, offset, len, buf);
8285
8286         if (buf != data)
8287                 kfree(buf);
8288
8289         return ret;
8290 }
8291
8292 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
8293 {
8294         struct tg3 *tp = netdev_priv(dev);
8295
8296         cmd->supported = (SUPPORTED_Autoneg);
8297
8298         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
8299                 cmd->supported |= (SUPPORTED_1000baseT_Half |
8300                                    SUPPORTED_1000baseT_Full);
8301
8302         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
8303                 cmd->supported |= (SUPPORTED_100baseT_Half |
8304                                   SUPPORTED_100baseT_Full |
8305                                   SUPPORTED_10baseT_Half |
8306                                   SUPPORTED_10baseT_Full |
8307                                   SUPPORTED_MII);
8308                 cmd->port = PORT_TP;
8309         } else {
8310                 cmd->supported |= SUPPORTED_FIBRE;
8311                 cmd->port = PORT_FIBRE;
8312         }
8313
8314         cmd->advertising = tp->link_config.advertising;
8315         if (netif_running(dev)) {
8316                 cmd->speed = tp->link_config.active_speed;
8317                 cmd->duplex = tp->link_config.active_duplex;
8318         }
8319         cmd->phy_address = PHY_ADDR;
8320         cmd->transceiver = 0;
8321         cmd->autoneg = tp->link_config.autoneg;
8322         cmd->maxtxpkt = 0;
8323         cmd->maxrxpkt = 0;
8324         return 0;
8325 }
8326
8327 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
8328 {
8329         struct tg3 *tp = netdev_priv(dev);
8330
8331         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) {
8332                 /* These are the only valid advertisement bits allowed.  */
8333                 if (cmd->autoneg == AUTONEG_ENABLE &&
8334                     (cmd->advertising & ~(ADVERTISED_1000baseT_Half |
8335                                           ADVERTISED_1000baseT_Full |
8336                                           ADVERTISED_Autoneg |
8337                                           ADVERTISED_FIBRE)))
8338                         return -EINVAL;
8339                 /* Fiber can only do SPEED_1000.  */
8340                 else if ((cmd->autoneg != AUTONEG_ENABLE) &&
8341                          (cmd->speed != SPEED_1000))
8342                         return -EINVAL;
8343         /* Copper cannot force SPEED_1000.  */
8344         } else if ((cmd->autoneg != AUTONEG_ENABLE) &&
8345                    (cmd->speed == SPEED_1000))
8346                 return -EINVAL;
8347         else if ((cmd->speed == SPEED_1000) &&
8348                  (tp->tg3_flags2 & TG3_FLAG_10_100_ONLY))
8349                 return -EINVAL;
8350
8351         tg3_full_lock(tp, 0);
8352
8353         tp->link_config.autoneg = cmd->autoneg;
8354         if (cmd->autoneg == AUTONEG_ENABLE) {
8355                 tp->link_config.advertising = (cmd->advertising |
8356                                               ADVERTISED_Autoneg);
8357                 tp->link_config.speed = SPEED_INVALID;
8358                 tp->link_config.duplex = DUPLEX_INVALID;
8359         } else {
8360                 tp->link_config.advertising = 0;
8361                 tp->link_config.speed = cmd->speed;
8362                 tp->link_config.duplex = cmd->duplex;
8363         }
8364
8365         tp->link_config.orig_speed = tp->link_config.speed;
8366         tp->link_config.orig_duplex = tp->link_config.duplex;
8367         tp->link_config.orig_autoneg = tp->link_config.autoneg;
8368
8369         if (netif_running(dev))
8370                 tg3_setup_phy(tp, 1);
8371
8372         tg3_full_unlock(tp);
8373
8374         return 0;
8375 }
8376
8377 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
8378 {
8379         struct tg3 *tp = netdev_priv(dev);
8380
8381         strcpy(info->driver, DRV_MODULE_NAME);
8382         strcpy(info->version, DRV_MODULE_VERSION);
8383         strcpy(info->fw_version, tp->fw_ver);
8384         strcpy(info->bus_info, pci_name(tp->pdev));
8385 }
8386
8387 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
8388 {
8389         struct tg3 *tp = netdev_priv(dev);
8390
8391         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
8392                 wol->supported = WAKE_MAGIC;
8393         else
8394                 wol->supported = 0;
8395         wol->wolopts = 0;
8396         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)
8397                 wol->wolopts = WAKE_MAGIC;
8398         memset(&wol->sopass, 0, sizeof(wol->sopass));
8399 }
8400
8401 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
8402 {
8403         struct tg3 *tp = netdev_priv(dev);
8404
8405         if (wol->wolopts & ~WAKE_MAGIC)
8406                 return -EINVAL;
8407         if ((wol->wolopts & WAKE_MAGIC) &&
8408             !(tp->tg3_flags & TG3_FLAG_WOL_CAP))
8409                 return -EINVAL;
8410
8411         spin_lock_bh(&tp->lock);
8412         if (wol->wolopts & WAKE_MAGIC)
8413                 tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
8414         else
8415                 tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE;
8416         spin_unlock_bh(&tp->lock);
8417
8418         return 0;
8419 }
8420
8421 static u32 tg3_get_msglevel(struct net_device *dev)
8422 {
8423         struct tg3 *tp = netdev_priv(dev);
8424         return tp->msg_enable;
8425 }
8426
8427 static void tg3_set_msglevel(struct net_device *dev, u32 value)
8428 {
8429         struct tg3 *tp = netdev_priv(dev);
8430         tp->msg_enable = value;
8431 }
8432
8433 static int tg3_set_tso(struct net_device *dev, u32 value)
8434 {
8435         struct tg3 *tp = netdev_priv(dev);
8436
8437         if (!(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
8438                 if (value)
8439                         return -EINVAL;
8440                 return 0;
8441         }
8442         if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO_2) &&
8443             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)) {
8444                 if (value) {
8445                         dev->features |= NETIF_F_TSO6;
8446                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
8447                                 dev->features |= NETIF_F_TSO_ECN;
8448                 } else
8449                         dev->features &= ~(NETIF_F_TSO6 | NETIF_F_TSO_ECN);
8450         }
8451         return ethtool_op_set_tso(dev, value);
8452 }
8453
8454 static int tg3_nway_reset(struct net_device *dev)
8455 {
8456         struct tg3 *tp = netdev_priv(dev);
8457         u32 bmcr;
8458         int r;
8459
8460         if (!netif_running(dev))
8461                 return -EAGAIN;
8462
8463         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
8464                 return -EINVAL;
8465
8466         spin_lock_bh(&tp->lock);
8467         r = -EINVAL;
8468         tg3_readphy(tp, MII_BMCR, &bmcr);
8469         if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
8470             ((bmcr & BMCR_ANENABLE) ||
8471              (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT))) {
8472                 tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
8473                                            BMCR_ANENABLE);
8474                 r = 0;
8475         }
8476         spin_unlock_bh(&tp->lock);
8477
8478         return r;
8479 }
8480
8481 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
8482 {
8483         struct tg3 *tp = netdev_priv(dev);
8484
8485         ering->rx_max_pending = TG3_RX_RING_SIZE - 1;
8486         ering->rx_mini_max_pending = 0;
8487         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
8488                 ering->rx_jumbo_max_pending = TG3_RX_JUMBO_RING_SIZE - 1;
8489         else
8490                 ering->rx_jumbo_max_pending = 0;
8491
8492         ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
8493
8494         ering->rx_pending = tp->rx_pending;
8495         ering->rx_mini_pending = 0;
8496         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
8497                 ering->rx_jumbo_pending = tp->rx_jumbo_pending;
8498         else
8499                 ering->rx_jumbo_pending = 0;
8500
8501         ering->tx_pending = tp->tx_pending;
8502 }
8503
8504 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
8505 {
8506         struct tg3 *tp = netdev_priv(dev);
8507         int irq_sync = 0, err = 0;
8508
8509         if ((ering->rx_pending > TG3_RX_RING_SIZE - 1) ||
8510             (ering->rx_jumbo_pending > TG3_RX_JUMBO_RING_SIZE - 1) ||
8511             (ering->tx_pending > TG3_TX_RING_SIZE - 1) ||
8512             (ering->tx_pending <= MAX_SKB_FRAGS) ||
8513             ((tp->tg3_flags2 & TG3_FLG2_TSO_BUG) &&
8514              (ering->tx_pending <= (MAX_SKB_FRAGS * 3))))
8515                 return -EINVAL;
8516
8517         if (netif_running(dev)) {
8518                 tg3_netif_stop(tp);
8519                 irq_sync = 1;
8520         }
8521
8522         tg3_full_lock(tp, irq_sync);
8523
8524         tp->rx_pending = ering->rx_pending;
8525
8526         if ((tp->tg3_flags2 & TG3_FLG2_MAX_RXPEND_64) &&
8527             tp->rx_pending > 63)
8528                 tp->rx_pending = 63;
8529         tp->rx_jumbo_pending = ering->rx_jumbo_pending;
8530         tp->tx_pending = ering->tx_pending;
8531
8532         if (netif_running(dev)) {
8533                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8534                 err = tg3_restart_hw(tp, 1);
8535                 if (!err)
8536                         tg3_netif_start(tp);
8537         }
8538
8539         tg3_full_unlock(tp);
8540
8541         return err;
8542 }
8543
8544 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
8545 {
8546         struct tg3 *tp = netdev_priv(dev);
8547
8548         epause->autoneg = (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) != 0;
8549         epause->rx_pause = (tp->tg3_flags & TG3_FLAG_RX_PAUSE) != 0;
8550         epause->tx_pause = (tp->tg3_flags & TG3_FLAG_TX_PAUSE) != 0;
8551 }
8552
8553 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
8554 {
8555         struct tg3 *tp = netdev_priv(dev);
8556         int irq_sync = 0, err = 0;
8557
8558         if (netif_running(dev)) {
8559                 tg3_netif_stop(tp);
8560                 irq_sync = 1;
8561         }
8562
8563         tg3_full_lock(tp, irq_sync);
8564
8565         if (epause->autoneg)
8566                 tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
8567         else
8568                 tp->tg3_flags &= ~TG3_FLAG_PAUSE_AUTONEG;
8569         if (epause->rx_pause)
8570                 tp->tg3_flags |= TG3_FLAG_RX_PAUSE;
8571         else
8572                 tp->tg3_flags &= ~TG3_FLAG_RX_PAUSE;
8573         if (epause->tx_pause)
8574                 tp->tg3_flags |= TG3_FLAG_TX_PAUSE;
8575         else
8576                 tp->tg3_flags &= ~TG3_FLAG_TX_PAUSE;
8577
8578         if (netif_running(dev)) {
8579                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8580                 err = tg3_restart_hw(tp, 1);
8581                 if (!err)
8582                         tg3_netif_start(tp);
8583         }
8584
8585         tg3_full_unlock(tp);
8586
8587         return err;
8588 }
8589
8590 static u32 tg3_get_rx_csum(struct net_device *dev)
8591 {
8592         struct tg3 *tp = netdev_priv(dev);
8593         return (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0;
8594 }
8595
8596 static int tg3_set_rx_csum(struct net_device *dev, u32 data)
8597 {
8598         struct tg3 *tp = netdev_priv(dev);
8599
8600         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
8601                 if (data != 0)
8602                         return -EINVAL;
8603                 return 0;
8604         }
8605
8606         spin_lock_bh(&tp->lock);
8607         if (data)
8608                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
8609         else
8610                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
8611         spin_unlock_bh(&tp->lock);
8612
8613         return 0;
8614 }
8615
8616 static int tg3_set_tx_csum(struct net_device *dev, u32 data)
8617 {
8618         struct tg3 *tp = netdev_priv(dev);
8619
8620         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
8621                 if (data != 0)
8622                         return -EINVAL;
8623                 return 0;
8624         }
8625
8626         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
8627             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
8628             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
8629             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
8630                 ethtool_op_set_tx_ipv6_csum(dev, data);
8631         else
8632                 ethtool_op_set_tx_csum(dev, data);
8633
8634         return 0;
8635 }
8636
8637 static int tg3_get_sset_count (struct net_device *dev, int sset)
8638 {
8639         switch (sset) {
8640         case ETH_SS_TEST:
8641                 return TG3_NUM_TEST;
8642         case ETH_SS_STATS:
8643                 return TG3_NUM_STATS;
8644         default:
8645                 return -EOPNOTSUPP;
8646         }
8647 }
8648
8649 static void tg3_get_strings (struct net_device *dev, u32 stringset, u8 *buf)
8650 {
8651         switch (stringset) {
8652         case ETH_SS_STATS:
8653                 memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
8654                 break;
8655         case ETH_SS_TEST:
8656                 memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
8657                 break;
8658         default:
8659                 WARN_ON(1);     /* we need a WARN() */
8660                 break;
8661         }
8662 }
8663
8664 static int tg3_phys_id(struct net_device *dev, u32 data)
8665 {
8666         struct tg3 *tp = netdev_priv(dev);
8667         int i;
8668
8669         if (!netif_running(tp->dev))
8670                 return -EAGAIN;
8671
8672         if (data == 0)
8673                 data = 2;
8674
8675         for (i = 0; i < (data * 2); i++) {
8676                 if ((i % 2) == 0)
8677                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8678                                            LED_CTRL_1000MBPS_ON |
8679                                            LED_CTRL_100MBPS_ON |
8680                                            LED_CTRL_10MBPS_ON |
8681                                            LED_CTRL_TRAFFIC_OVERRIDE |
8682                                            LED_CTRL_TRAFFIC_BLINK |
8683                                            LED_CTRL_TRAFFIC_LED);
8684
8685                 else
8686                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8687                                            LED_CTRL_TRAFFIC_OVERRIDE);
8688
8689                 if (msleep_interruptible(500))
8690                         break;
8691         }
8692         tw32(MAC_LED_CTRL, tp->led_ctrl);
8693         return 0;
8694 }
8695
8696 static void tg3_get_ethtool_stats (struct net_device *dev,
8697                                    struct ethtool_stats *estats, u64 *tmp_stats)
8698 {
8699         struct tg3 *tp = netdev_priv(dev);
8700         memcpy(tmp_stats, tg3_get_estats(tp), sizeof(tp->estats));
8701 }
8702
8703 #define NVRAM_TEST_SIZE 0x100
8704 #define NVRAM_SELFBOOT_FORMAT1_0_SIZE   0x14
8705 #define NVRAM_SELFBOOT_FORMAT1_2_SIZE   0x18
8706 #define NVRAM_SELFBOOT_FORMAT1_3_SIZE   0x1c
8707 #define NVRAM_SELFBOOT_HW_SIZE 0x20
8708 #define NVRAM_SELFBOOT_DATA_SIZE 0x1c
8709
8710 static int tg3_test_nvram(struct tg3 *tp)
8711 {
8712         u32 *buf, csum, magic;
8713         int i, j, k, err = 0, size;
8714
8715         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
8716                 return -EIO;
8717
8718         if (magic == TG3_EEPROM_MAGIC)
8719                 size = NVRAM_TEST_SIZE;
8720         else if ((magic & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW) {
8721                 if ((magic & TG3_EEPROM_SB_FORMAT_MASK) ==
8722                     TG3_EEPROM_SB_FORMAT_1) {
8723                         switch (magic & TG3_EEPROM_SB_REVISION_MASK) {
8724                         case TG3_EEPROM_SB_REVISION_0:
8725                                 size = NVRAM_SELFBOOT_FORMAT1_0_SIZE;
8726                                 break;
8727                         case TG3_EEPROM_SB_REVISION_2:
8728                                 size = NVRAM_SELFBOOT_FORMAT1_2_SIZE;
8729                                 break;
8730                         case TG3_EEPROM_SB_REVISION_3:
8731                                 size = NVRAM_SELFBOOT_FORMAT1_3_SIZE;
8732                                 break;
8733                         default:
8734                                 return 0;
8735                         }
8736                 } else
8737                         return 0;
8738         } else if ((magic & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
8739                 size = NVRAM_SELFBOOT_HW_SIZE;
8740         else
8741                 return -EIO;
8742
8743         buf = kmalloc(size, GFP_KERNEL);
8744         if (buf == NULL)
8745                 return -ENOMEM;
8746
8747         err = -EIO;
8748         for (i = 0, j = 0; i < size; i += 4, j++) {
8749                 u32 val;
8750
8751                 if ((err = tg3_nvram_read(tp, i, &val)) != 0)
8752                         break;
8753                 buf[j] = cpu_to_le32(val);
8754         }
8755         if (i < size)
8756                 goto out;
8757
8758         /* Selfboot format */
8759         if ((cpu_to_be32(buf[0]) & TG3_EEPROM_MAGIC_FW_MSK) ==
8760             TG3_EEPROM_MAGIC_FW) {
8761                 u8 *buf8 = (u8 *) buf, csum8 = 0;
8762
8763                 if ((cpu_to_be32(buf[0]) & TG3_EEPROM_SB_REVISION_MASK) ==
8764                     TG3_EEPROM_SB_REVISION_2) {
8765                         /* For rev 2, the csum doesn't include the MBA. */
8766                         for (i = 0; i < TG3_EEPROM_SB_F1R2_MBA_OFF; i++)
8767                                 csum8 += buf8[i];
8768                         for (i = TG3_EEPROM_SB_F1R2_MBA_OFF + 4; i < size; i++)
8769                                 csum8 += buf8[i];
8770                 } else {
8771                         for (i = 0; i < size; i++)
8772                                 csum8 += buf8[i];
8773                 }
8774
8775                 if (csum8 == 0) {
8776                         err = 0;
8777                         goto out;
8778                 }
8779
8780                 err = -EIO;
8781                 goto out;
8782         }
8783
8784         if ((cpu_to_be32(buf[0]) & TG3_EEPROM_MAGIC_HW_MSK) ==
8785             TG3_EEPROM_MAGIC_HW) {
8786                 u8 data[NVRAM_SELFBOOT_DATA_SIZE];
8787                 u8 parity[NVRAM_SELFBOOT_DATA_SIZE];
8788                 u8 *buf8 = (u8 *) buf;
8789
8790                 /* Separate the parity bits and the data bytes.  */
8791                 for (i = 0, j = 0, k = 0; i < NVRAM_SELFBOOT_HW_SIZE; i++) {
8792                         if ((i == 0) || (i == 8)) {
8793                                 int l;
8794                                 u8 msk;
8795
8796                                 for (l = 0, msk = 0x80; l < 7; l++, msk >>= 1)
8797                                         parity[k++] = buf8[i] & msk;
8798                                 i++;
8799                         }
8800                         else if (i == 16) {
8801                                 int l;
8802                                 u8 msk;
8803
8804                                 for (l = 0, msk = 0x20; l < 6; l++, msk >>= 1)
8805                                         parity[k++] = buf8[i] & msk;
8806                                 i++;
8807
8808                                 for (l = 0, msk = 0x80; l < 8; l++, msk >>= 1)
8809                                         parity[k++] = buf8[i] & msk;
8810                                 i++;
8811                         }
8812                         data[j++] = buf8[i];
8813                 }
8814
8815                 err = -EIO;
8816                 for (i = 0; i < NVRAM_SELFBOOT_DATA_SIZE; i++) {
8817                         u8 hw8 = hweight8(data[i]);
8818
8819                         if ((hw8 & 0x1) && parity[i])
8820                                 goto out;
8821                         else if (!(hw8 & 0x1) && !parity[i])
8822                                 goto out;
8823                 }
8824                 err = 0;
8825                 goto out;
8826         }
8827
8828         /* Bootstrap checksum at offset 0x10 */
8829         csum = calc_crc((unsigned char *) buf, 0x10);
8830         if(csum != cpu_to_le32(buf[0x10/4]))
8831                 goto out;
8832
8833         /* Manufacturing block starts at offset 0x74, checksum at 0xfc */
8834         csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
8835         if (csum != cpu_to_le32(buf[0xfc/4]))
8836                  goto out;
8837
8838         err = 0;
8839
8840 out:
8841         kfree(buf);
8842         return err;
8843 }
8844
8845 #define TG3_SERDES_TIMEOUT_SEC  2
8846 #define TG3_COPPER_TIMEOUT_SEC  6
8847
8848 static int tg3_test_link(struct tg3 *tp)
8849 {
8850         int i, max;
8851
8852         if (!netif_running(tp->dev))
8853                 return -ENODEV;
8854
8855         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
8856                 max = TG3_SERDES_TIMEOUT_SEC;
8857         else
8858                 max = TG3_COPPER_TIMEOUT_SEC;
8859
8860         for (i = 0; i < max; i++) {
8861                 if (netif_carrier_ok(tp->dev))
8862                         return 0;
8863
8864                 if (msleep_interruptible(1000))
8865                         break;
8866         }
8867
8868         return -EIO;
8869 }
8870
8871 /* Only test the commonly used registers */
8872 static int tg3_test_registers(struct tg3 *tp)
8873 {
8874         int i, is_5705, is_5750;
8875         u32 offset, read_mask, write_mask, val, save_val, read_val;
8876         static struct {
8877                 u16 offset;
8878                 u16 flags;
8879 #define TG3_FL_5705     0x1
8880 #define TG3_FL_NOT_5705 0x2
8881 #define TG3_FL_NOT_5788 0x4
8882 #define TG3_FL_NOT_5750 0x8
8883                 u32 read_mask;
8884                 u32 write_mask;
8885         } reg_tbl[] = {
8886                 /* MAC Control Registers */
8887                 { MAC_MODE, TG3_FL_NOT_5705,
8888                         0x00000000, 0x00ef6f8c },
8889                 { MAC_MODE, TG3_FL_5705,
8890                         0x00000000, 0x01ef6b8c },
8891                 { MAC_STATUS, TG3_FL_NOT_5705,
8892                         0x03800107, 0x00000000 },
8893                 { MAC_STATUS, TG3_FL_5705,
8894                         0x03800100, 0x00000000 },
8895                 { MAC_ADDR_0_HIGH, 0x0000,
8896                         0x00000000, 0x0000ffff },
8897                 { MAC_ADDR_0_LOW, 0x0000,
8898                         0x00000000, 0xffffffff },
8899                 { MAC_RX_MTU_SIZE, 0x0000,
8900                         0x00000000, 0x0000ffff },
8901                 { MAC_TX_MODE, 0x0000,
8902                         0x00000000, 0x00000070 },
8903                 { MAC_TX_LENGTHS, 0x0000,
8904                         0x00000000, 0x00003fff },
8905                 { MAC_RX_MODE, TG3_FL_NOT_5705,
8906                         0x00000000, 0x000007fc },
8907                 { MAC_RX_MODE, TG3_FL_5705,
8908                         0x00000000, 0x000007dc },
8909                 { MAC_HASH_REG_0, 0x0000,
8910                         0x00000000, 0xffffffff },
8911                 { MAC_HASH_REG_1, 0x0000,
8912                         0x00000000, 0xffffffff },
8913                 { MAC_HASH_REG_2, 0x0000,
8914                         0x00000000, 0xffffffff },
8915                 { MAC_HASH_REG_3, 0x0000,
8916                         0x00000000, 0xffffffff },
8917
8918                 /* Receive Data and Receive BD Initiator Control Registers. */
8919                 { RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
8920                         0x00000000, 0xffffffff },
8921                 { RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
8922                         0x00000000, 0xffffffff },
8923                 { RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
8924                         0x00000000, 0x00000003 },
8925                 { RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
8926                         0x00000000, 0xffffffff },
8927                 { RCVDBDI_STD_BD+0, 0x0000,
8928                         0x00000000, 0xffffffff },
8929                 { RCVDBDI_STD_BD+4, 0x0000,
8930                         0x00000000, 0xffffffff },
8931                 { RCVDBDI_STD_BD+8, 0x0000,
8932                         0x00000000, 0xffff0002 },
8933                 { RCVDBDI_STD_BD+0xc, 0x0000,
8934                         0x00000000, 0xffffffff },
8935
8936                 /* Receive BD Initiator Control Registers. */
8937                 { RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
8938                         0x00000000, 0xffffffff },
8939                 { RCVBDI_STD_THRESH, TG3_FL_5705,
8940                         0x00000000, 0x000003ff },
8941                 { RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
8942                         0x00000000, 0xffffffff },
8943
8944                 /* Host Coalescing Control Registers. */
8945                 { HOSTCC_MODE, TG3_FL_NOT_5705,
8946                         0x00000000, 0x00000004 },
8947                 { HOSTCC_MODE, TG3_FL_5705,
8948                         0x00000000, 0x000000f6 },
8949                 { HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
8950                         0x00000000, 0xffffffff },
8951                 { HOSTCC_RXCOL_TICKS, TG3_FL_5705,
8952                         0x00000000, 0x000003ff },
8953                 { HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
8954                         0x00000000, 0xffffffff },
8955                 { HOSTCC_TXCOL_TICKS, TG3_FL_5705,
8956                         0x00000000, 0x000003ff },
8957                 { HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
8958                         0x00000000, 0xffffffff },
8959                 { HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
8960                         0x00000000, 0x000000ff },
8961                 { HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
8962                         0x00000000, 0xffffffff },
8963                 { HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
8964                         0x00000000, 0x000000ff },
8965                 { HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
8966                         0x00000000, 0xffffffff },
8967                 { HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
8968                         0x00000000, 0xffffffff },
8969                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
8970                         0x00000000, 0xffffffff },
8971                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
8972                         0x00000000, 0x000000ff },
8973                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
8974                         0x00000000, 0xffffffff },
8975                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
8976                         0x00000000, 0x000000ff },
8977                 { HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
8978                         0x00000000, 0xffffffff },
8979                 { HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
8980                         0x00000000, 0xffffffff },
8981                 { HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
8982                         0x00000000, 0xffffffff },
8983                 { HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
8984                         0x00000000, 0xffffffff },
8985                 { HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
8986                         0x00000000, 0xffffffff },
8987                 { HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
8988                         0xffffffff, 0x00000000 },
8989                 { HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
8990                         0xffffffff, 0x00000000 },
8991
8992                 /* Buffer Manager Control Registers. */
8993                 { BUFMGR_MB_POOL_ADDR, TG3_FL_NOT_5750,
8994                         0x00000000, 0x007fff80 },
8995                 { BUFMGR_MB_POOL_SIZE, TG3_FL_NOT_5750,
8996                         0x00000000, 0x007fffff },
8997                 { BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
8998                         0x00000000, 0x0000003f },
8999                 { BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
9000                         0x00000000, 0x000001ff },
9001                 { BUFMGR_MB_HIGH_WATER, 0x0000,
9002                         0x00000000, 0x000001ff },
9003                 { BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
9004                         0xffffffff, 0x00000000 },
9005                 { BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
9006                         0xffffffff, 0x00000000 },
9007
9008                 /* Mailbox Registers */
9009                 { GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
9010                         0x00000000, 0x000001ff },
9011                 { GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
9012                         0x00000000, 0x000001ff },
9013                 { GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
9014                         0x00000000, 0x000007ff },
9015                 { GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
9016                         0x00000000, 0x000001ff },
9017
9018                 { 0xffff, 0x0000, 0x00000000, 0x00000000 },
9019         };
9020
9021         is_5705 = is_5750 = 0;
9022         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
9023                 is_5705 = 1;
9024                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
9025                         is_5750 = 1;
9026         }
9027
9028         for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
9029                 if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
9030                         continue;
9031
9032                 if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
9033                         continue;
9034
9035                 if ((tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
9036                     (reg_tbl[i].flags & TG3_FL_NOT_5788))
9037                         continue;
9038
9039                 if (is_5750 && (reg_tbl[i].flags & TG3_FL_NOT_5750))
9040                         continue;
9041
9042                 offset = (u32) reg_tbl[i].offset;
9043                 read_mask = reg_tbl[i].read_mask;
9044                 write_mask = reg_tbl[i].write_mask;
9045
9046                 /* Save the original register content */
9047                 save_val = tr32(offset);
9048
9049                 /* Determine the read-only value. */
9050                 read_val = save_val & read_mask;
9051
9052                 /* Write zero to the register, then make sure the read-only bits
9053                  * are not changed and the read/write bits are all zeros.
9054                  */
9055                 tw32(offset, 0);
9056
9057                 val = tr32(offset);
9058
9059                 /* Test the read-only and read/write bits. */
9060                 if (((val & read_mask) != read_val) || (val & write_mask))
9061                         goto out;
9062
9063                 /* Write ones to all the bits defined by RdMask and WrMask, then
9064                  * make sure the read-only bits are not changed and the
9065                  * read/write bits are all ones.
9066                  */
9067                 tw32(offset, read_mask | write_mask);
9068
9069                 val = tr32(offset);
9070
9071                 /* Test the read-only bits. */
9072                 if ((val & read_mask) != read_val)
9073                         goto out;
9074
9075                 /* Test the read/write bits. */
9076                 if ((val & write_mask) != write_mask)
9077                         goto out;
9078
9079                 tw32(offset, save_val);
9080         }
9081
9082         return 0;
9083
9084 out:
9085         if (netif_msg_hw(tp))
9086                 printk(KERN_ERR PFX "Register test failed at offset %x\n",
9087                        offset);
9088         tw32(offset, save_val);
9089         return -EIO;
9090 }
9091
9092 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
9093 {
9094         static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
9095         int i;
9096         u32 j;
9097
9098         for (i = 0; i < ARRAY_SIZE(test_pattern); i++) {
9099                 for (j = 0; j < len; j += 4) {
9100                         u32 val;
9101
9102                         tg3_write_mem(tp, offset + j, test_pattern[i]);
9103                         tg3_read_mem(tp, offset + j, &val);
9104                         if (val != test_pattern[i])
9105                                 return -EIO;
9106                 }
9107         }
9108         return 0;
9109 }
9110
9111 static int tg3_test_memory(struct tg3 *tp)
9112 {
9113         static struct mem_entry {
9114                 u32 offset;
9115                 u32 len;
9116         } mem_tbl_570x[] = {
9117                 { 0x00000000, 0x00b50},
9118                 { 0x00002000, 0x1c000},
9119                 { 0xffffffff, 0x00000}
9120         }, mem_tbl_5705[] = {
9121                 { 0x00000100, 0x0000c},
9122                 { 0x00000200, 0x00008},
9123                 { 0x00004000, 0x00800},
9124                 { 0x00006000, 0x01000},
9125                 { 0x00008000, 0x02000},
9126                 { 0x00010000, 0x0e000},
9127                 { 0xffffffff, 0x00000}
9128         }, mem_tbl_5755[] = {
9129                 { 0x00000200, 0x00008},
9130                 { 0x00004000, 0x00800},
9131                 { 0x00006000, 0x00800},
9132                 { 0x00008000, 0x02000},
9133                 { 0x00010000, 0x0c000},
9134                 { 0xffffffff, 0x00000}
9135         }, mem_tbl_5906[] = {
9136                 { 0x00000200, 0x00008},
9137                 { 0x00004000, 0x00400},
9138                 { 0x00006000, 0x00400},
9139                 { 0x00008000, 0x01000},
9140                 { 0x00010000, 0x01000},
9141                 { 0xffffffff, 0x00000}
9142         };
9143         struct mem_entry *mem_tbl;
9144         int err = 0;
9145         int i;
9146
9147         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
9148                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
9149                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
9150                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
9151                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
9152                         mem_tbl = mem_tbl_5755;
9153                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
9154                         mem_tbl = mem_tbl_5906;
9155                 else
9156                         mem_tbl = mem_tbl_5705;
9157         } else
9158                 mem_tbl = mem_tbl_570x;
9159
9160         for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
9161                 if ((err = tg3_do_mem_test(tp, mem_tbl[i].offset,
9162                     mem_tbl[i].len)) != 0)
9163                         break;
9164         }
9165
9166         return err;
9167 }
9168
9169 #define TG3_MAC_LOOPBACK        0
9170 #define TG3_PHY_LOOPBACK        1
9171
9172 static int tg3_run_loopback(struct tg3 *tp, int loopback_mode)
9173 {
9174         u32 mac_mode, rx_start_idx, rx_idx, tx_idx, opaque_key;
9175         u32 desc_idx;
9176         struct sk_buff *skb, *rx_skb;
9177         u8 *tx_data;
9178         dma_addr_t map;
9179         int num_pkts, tx_len, rx_len, i, err;
9180         struct tg3_rx_buffer_desc *desc;
9181
9182         if (loopback_mode == TG3_MAC_LOOPBACK) {
9183                 /* HW errata - mac loopback fails in some cases on 5780.
9184                  * Normal traffic and PHY loopback are not affected by
9185                  * errata.
9186                  */
9187                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780)
9188                         return 0;
9189
9190                 mac_mode = (tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK) |
9191                            MAC_MODE_PORT_INT_LPBACK;
9192                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
9193                         mac_mode |= MAC_MODE_LINK_POLARITY;
9194                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
9195                         mac_mode |= MAC_MODE_PORT_MODE_MII;
9196                 else
9197                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
9198                 tw32(MAC_MODE, mac_mode);
9199         } else if (loopback_mode == TG3_PHY_LOOPBACK) {
9200                 u32 val;
9201
9202                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
9203                         u32 phytest;
9204
9205                         if (!tg3_readphy(tp, MII_TG3_EPHY_TEST, &phytest)) {
9206                                 u32 phy;
9207
9208                                 tg3_writephy(tp, MII_TG3_EPHY_TEST,
9209                                              phytest | MII_TG3_EPHY_SHADOW_EN);
9210                                 if (!tg3_readphy(tp, 0x1b, &phy))
9211                                         tg3_writephy(tp, 0x1b, phy & ~0x20);
9212                                 tg3_writephy(tp, MII_TG3_EPHY_TEST, phytest);
9213                         }
9214                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED100;
9215                 } else
9216                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED1000;
9217
9218                 tg3_phy_toggle_automdix(tp, 0);
9219
9220                 tg3_writephy(tp, MII_BMCR, val);
9221                 udelay(40);
9222
9223                 mac_mode = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
9224                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
9225                         tg3_writephy(tp, MII_TG3_EPHY_PTEST, 0x1800);
9226                         mac_mode |= MAC_MODE_PORT_MODE_MII;
9227                 } else
9228                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
9229
9230                 /* reset to prevent losing 1st rx packet intermittently */
9231                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
9232                         tw32_f(MAC_RX_MODE, RX_MODE_RESET);
9233                         udelay(10);
9234                         tw32_f(MAC_RX_MODE, tp->rx_mode);
9235                 }
9236                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
9237                         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)
9238                                 mac_mode &= ~MAC_MODE_LINK_POLARITY;
9239                         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411)
9240                                 mac_mode |= MAC_MODE_LINK_POLARITY;
9241                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
9242                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
9243                 }
9244                 tw32(MAC_MODE, mac_mode);
9245         }
9246         else
9247                 return -EINVAL;
9248
9249         err = -EIO;
9250
9251         tx_len = 1514;
9252         skb = netdev_alloc_skb(tp->dev, tx_len);
9253         if (!skb)
9254                 return -ENOMEM;
9255
9256         tx_data = skb_put(skb, tx_len);
9257         memcpy(tx_data, tp->dev->dev_addr, 6);
9258         memset(tx_data + 6, 0x0, 8);
9259
9260         tw32(MAC_RX_MTU_SIZE, tx_len + 4);
9261
9262         for (i = 14; i < tx_len; i++)
9263                 tx_data[i] = (u8) (i & 0xff);
9264
9265         map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
9266
9267         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
9268              HOSTCC_MODE_NOW);
9269
9270         udelay(10);
9271
9272         rx_start_idx = tp->hw_status->idx[0].rx_producer;
9273
9274         num_pkts = 0;
9275
9276         tg3_set_txd(tp, tp->tx_prod, map, tx_len, 0, 1);
9277
9278         tp->tx_prod++;
9279         num_pkts++;
9280
9281         tw32_tx_mbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW,
9282                      tp->tx_prod);
9283         tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW);
9284
9285         udelay(10);
9286
9287         /* 250 usec to allow enough time on some 10/100 Mbps devices.  */
9288         for (i = 0; i < 25; i++) {
9289                 tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
9290                        HOSTCC_MODE_NOW);
9291
9292                 udelay(10);
9293
9294                 tx_idx = tp->hw_status->idx[0].tx_consumer;
9295                 rx_idx = tp->hw_status->idx[0].rx_producer;
9296                 if ((tx_idx == tp->tx_prod) &&
9297                     (rx_idx == (rx_start_idx + num_pkts)))
9298                         break;
9299         }
9300
9301         pci_unmap_single(tp->pdev, map, tx_len, PCI_DMA_TODEVICE);
9302         dev_kfree_skb(skb);
9303
9304         if (tx_idx != tp->tx_prod)
9305                 goto out;
9306
9307         if (rx_idx != rx_start_idx + num_pkts)
9308                 goto out;
9309
9310         desc = &tp->rx_rcb[rx_start_idx];
9311         desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
9312         opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
9313         if (opaque_key != RXD_OPAQUE_RING_STD)
9314                 goto out;
9315
9316         if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
9317             (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
9318                 goto out;
9319
9320         rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4;
9321         if (rx_len != tx_len)
9322                 goto out;
9323
9324         rx_skb = tp->rx_std_buffers[desc_idx].skb;
9325
9326         map = pci_unmap_addr(&tp->rx_std_buffers[desc_idx], mapping);
9327         pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len, PCI_DMA_FROMDEVICE);
9328
9329         for (i = 14; i < tx_len; i++) {
9330                 if (*(rx_skb->data + i) != (u8) (i & 0xff))
9331                         goto out;
9332         }
9333         err = 0;
9334
9335         /* tg3_free_rings will unmap and free the rx_skb */
9336 out:
9337         return err;
9338 }
9339
9340 #define TG3_MAC_LOOPBACK_FAILED         1
9341 #define TG3_PHY_LOOPBACK_FAILED         2
9342 #define TG3_LOOPBACK_FAILED             (TG3_MAC_LOOPBACK_FAILED |      \
9343                                          TG3_PHY_LOOPBACK_FAILED)
9344
9345 static int tg3_test_loopback(struct tg3 *tp)
9346 {
9347         int err = 0;
9348         u32 cpmuctrl = 0;
9349
9350         if (!netif_running(tp->dev))
9351                 return TG3_LOOPBACK_FAILED;
9352
9353         err = tg3_reset_hw(tp, 1);
9354         if (err)
9355                 return TG3_LOOPBACK_FAILED;
9356
9357         if (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) {
9358                 int i;
9359                 u32 status;
9360
9361                 tw32(TG3_CPMU_MUTEX_REQ, CPMU_MUTEX_REQ_DRIVER);
9362
9363                 /* Wait for up to 40 microseconds to acquire lock. */
9364                 for (i = 0; i < 4; i++) {
9365                         status = tr32(TG3_CPMU_MUTEX_GNT);
9366                         if (status == CPMU_MUTEX_GNT_DRIVER)
9367                                 break;
9368                         udelay(10);
9369                 }
9370
9371                 if (status != CPMU_MUTEX_GNT_DRIVER)
9372                         return TG3_LOOPBACK_FAILED;
9373
9374                 cpmuctrl = tr32(TG3_CPMU_CTRL);
9375
9376                 /* Turn off power management based on link speed. */
9377                 tw32(TG3_CPMU_CTRL,
9378                      cpmuctrl & ~CPMU_CTRL_LINK_SPEED_MODE);
9379         }
9380
9381         if (tg3_run_loopback(tp, TG3_MAC_LOOPBACK))
9382                 err |= TG3_MAC_LOOPBACK_FAILED;
9383
9384         if (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) {
9385                 tw32(TG3_CPMU_CTRL, cpmuctrl);
9386
9387                 /* Release the mutex */
9388                 tw32(TG3_CPMU_MUTEX_GNT, CPMU_MUTEX_GNT_DRIVER);
9389         }
9390
9391         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
9392                 if (tg3_run_loopback(tp, TG3_PHY_LOOPBACK))
9393                         err |= TG3_PHY_LOOPBACK_FAILED;
9394         }
9395
9396         return err;
9397 }
9398
9399 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
9400                           u64 *data)
9401 {
9402         struct tg3 *tp = netdev_priv(dev);
9403
9404         if (tp->link_config.phy_is_low_power)
9405                 tg3_set_power_state(tp, PCI_D0);
9406
9407         memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
9408
9409         if (tg3_test_nvram(tp) != 0) {
9410                 etest->flags |= ETH_TEST_FL_FAILED;
9411                 data[0] = 1;
9412         }
9413         if (tg3_test_link(tp) != 0) {
9414                 etest->flags |= ETH_TEST_FL_FAILED;
9415                 data[1] = 1;
9416         }
9417         if (etest->flags & ETH_TEST_FL_OFFLINE) {
9418                 int err, irq_sync = 0;
9419
9420                 if (netif_running(dev)) {
9421                         tg3_netif_stop(tp);
9422                         irq_sync = 1;
9423                 }
9424
9425                 tg3_full_lock(tp, irq_sync);
9426
9427                 tg3_halt(tp, RESET_KIND_SUSPEND, 1);
9428                 err = tg3_nvram_lock(tp);
9429                 tg3_halt_cpu(tp, RX_CPU_BASE);
9430                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
9431                         tg3_halt_cpu(tp, TX_CPU_BASE);
9432                 if (!err)
9433                         tg3_nvram_unlock(tp);
9434
9435                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
9436                         tg3_phy_reset(tp);
9437
9438                 if (tg3_test_registers(tp) != 0) {
9439                         etest->flags |= ETH_TEST_FL_FAILED;
9440                         data[2] = 1;
9441                 }
9442                 if (tg3_test_memory(tp) != 0) {
9443                         etest->flags |= ETH_TEST_FL_FAILED;
9444                         data[3] = 1;
9445                 }
9446                 if ((data[4] = tg3_test_loopback(tp)) != 0)
9447                         etest->flags |= ETH_TEST_FL_FAILED;
9448
9449                 tg3_full_unlock(tp);
9450
9451                 if (tg3_test_interrupt(tp) != 0) {
9452                         etest->flags |= ETH_TEST_FL_FAILED;
9453                         data[5] = 1;
9454                 }
9455
9456                 tg3_full_lock(tp, 0);
9457
9458                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9459                 if (netif_running(dev)) {
9460                         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
9461                         if (!tg3_restart_hw(tp, 1))
9462                                 tg3_netif_start(tp);
9463                 }
9464
9465                 tg3_full_unlock(tp);
9466         }
9467         if (tp->link_config.phy_is_low_power)
9468                 tg3_set_power_state(tp, PCI_D3hot);
9469
9470 }
9471
9472 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9473 {
9474         struct mii_ioctl_data *data = if_mii(ifr);
9475         struct tg3 *tp = netdev_priv(dev);
9476         int err;
9477
9478         switch(cmd) {
9479         case SIOCGMIIPHY:
9480                 data->phy_id = PHY_ADDR;
9481
9482                 /* fallthru */
9483         case SIOCGMIIREG: {
9484                 u32 mii_regval;
9485
9486                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
9487                         break;                  /* We have no PHY */
9488
9489                 if (tp->link_config.phy_is_low_power)
9490                         return -EAGAIN;
9491
9492                 spin_lock_bh(&tp->lock);
9493                 err = tg3_readphy(tp, data->reg_num & 0x1f, &mii_regval);
9494                 spin_unlock_bh(&tp->lock);
9495
9496                 data->val_out = mii_regval;
9497
9498                 return err;
9499         }
9500
9501         case SIOCSMIIREG:
9502                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
9503                         break;                  /* We have no PHY */
9504
9505                 if (!capable(CAP_NET_ADMIN))
9506                         return -EPERM;
9507
9508                 if (tp->link_config.phy_is_low_power)
9509                         return -EAGAIN;
9510
9511                 spin_lock_bh(&tp->lock);
9512                 err = tg3_writephy(tp, data->reg_num & 0x1f, data->val_in);
9513                 spin_unlock_bh(&tp->lock);
9514
9515                 return err;
9516
9517         default:
9518                 /* do nothing */
9519                 break;
9520         }
9521         return -EOPNOTSUPP;
9522 }
9523
9524 #if TG3_VLAN_TAG_USED
9525 static void tg3_vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
9526 {
9527         struct tg3 *tp = netdev_priv(dev);
9528
9529         if (netif_running(dev))
9530                 tg3_netif_stop(tp);
9531
9532         tg3_full_lock(tp, 0);
9533
9534         tp->vlgrp = grp;
9535
9536         /* Update RX_MODE_KEEP_VLAN_TAG bit in RX_MODE register. */
9537         __tg3_set_rx_mode(dev);
9538
9539         if (netif_running(dev))
9540                 tg3_netif_start(tp);
9541
9542         tg3_full_unlock(tp);
9543 }
9544 #endif
9545
9546 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
9547 {
9548         struct tg3 *tp = netdev_priv(dev);
9549
9550         memcpy(ec, &tp->coal, sizeof(*ec));
9551         return 0;
9552 }
9553
9554 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
9555 {
9556         struct tg3 *tp = netdev_priv(dev);
9557         u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
9558         u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
9559
9560         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
9561                 max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
9562                 max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
9563                 max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
9564                 min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
9565         }
9566
9567         if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
9568             (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
9569             (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
9570             (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
9571             (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
9572             (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
9573             (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
9574             (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
9575             (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
9576             (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
9577                 return -EINVAL;
9578
9579         /* No rx interrupts will be generated if both are zero */
9580         if ((ec->rx_coalesce_usecs == 0) &&
9581             (ec->rx_max_coalesced_frames == 0))
9582                 return -EINVAL;
9583
9584         /* No tx interrupts will be generated if both are zero */
9585         if ((ec->tx_coalesce_usecs == 0) &&
9586             (ec->tx_max_coalesced_frames == 0))
9587                 return -EINVAL;
9588
9589         /* Only copy relevant parameters, ignore all others. */
9590         tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
9591         tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
9592         tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
9593         tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
9594         tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
9595         tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
9596         tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
9597         tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
9598         tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
9599
9600         if (netif_running(dev)) {
9601                 tg3_full_lock(tp, 0);
9602                 __tg3_set_coalesce(tp, &tp->coal);
9603                 tg3_full_unlock(tp);
9604         }
9605         return 0;
9606 }
9607
9608 static const struct ethtool_ops tg3_ethtool_ops = {
9609         .get_settings           = tg3_get_settings,
9610         .set_settings           = tg3_set_settings,
9611         .get_drvinfo            = tg3_get_drvinfo,
9612         .get_regs_len           = tg3_get_regs_len,
9613         .get_regs               = tg3_get_regs,
9614         .get_wol                = tg3_get_wol,
9615         .set_wol                = tg3_set_wol,
9616         .get_msglevel           = tg3_get_msglevel,
9617         .set_msglevel           = tg3_set_msglevel,
9618         .nway_reset             = tg3_nway_reset,
9619         .get_link               = ethtool_op_get_link,
9620         .get_eeprom_len         = tg3_get_eeprom_len,
9621         .get_eeprom             = tg3_get_eeprom,
9622         .set_eeprom             = tg3_set_eeprom,
9623         .get_ringparam          = tg3_get_ringparam,
9624         .set_ringparam          = tg3_set_ringparam,
9625         .get_pauseparam         = tg3_get_pauseparam,
9626         .set_pauseparam         = tg3_set_pauseparam,
9627         .get_rx_csum            = tg3_get_rx_csum,
9628         .set_rx_csum            = tg3_set_rx_csum,
9629         .set_tx_csum            = tg3_set_tx_csum,
9630         .set_sg                 = ethtool_op_set_sg,
9631         .set_tso                = tg3_set_tso,
9632         .self_test              = tg3_self_test,
9633         .get_strings            = tg3_get_strings,
9634         .phys_id                = tg3_phys_id,
9635         .get_ethtool_stats      = tg3_get_ethtool_stats,
9636         .get_coalesce           = tg3_get_coalesce,
9637         .set_coalesce           = tg3_set_coalesce,
9638         .get_sset_count         = tg3_get_sset_count,
9639 };
9640
9641 static void __devinit tg3_get_eeprom_size(struct tg3 *tp)
9642 {
9643         u32 cursize, val, magic;
9644
9645         tp->nvram_size = EEPROM_CHIP_SIZE;
9646
9647         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
9648                 return;
9649
9650         if ((magic != TG3_EEPROM_MAGIC) &&
9651             ((magic & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW) &&
9652             ((magic & TG3_EEPROM_MAGIC_HW_MSK) != TG3_EEPROM_MAGIC_HW))
9653                 return;
9654
9655         /*
9656          * Size the chip by reading offsets at increasing powers of two.
9657          * When we encounter our validation signature, we know the addressing
9658          * has wrapped around, and thus have our chip size.
9659          */
9660         cursize = 0x10;
9661
9662         while (cursize < tp->nvram_size) {
9663                 if (tg3_nvram_read_swab(tp, cursize, &val) != 0)
9664                         return;
9665
9666                 if (val == magic)
9667                         break;
9668
9669                 cursize <<= 1;
9670         }
9671
9672         tp->nvram_size = cursize;
9673 }
9674
9675 static void __devinit tg3_get_nvram_size(struct tg3 *tp)
9676 {
9677         u32 val;
9678
9679         if (tg3_nvram_read_swab(tp, 0, &val) != 0)
9680                 return;
9681
9682         /* Selfboot format */
9683         if (val != TG3_EEPROM_MAGIC) {
9684                 tg3_get_eeprom_size(tp);
9685                 return;
9686         }
9687
9688         if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
9689                 if (val != 0) {
9690                         tp->nvram_size = (val >> 16) * 1024;
9691                         return;
9692                 }
9693         }
9694         tp->nvram_size = 0x80000;
9695 }
9696
9697 static void __devinit tg3_get_nvram_info(struct tg3 *tp)
9698 {
9699         u32 nvcfg1;
9700
9701         nvcfg1 = tr32(NVRAM_CFG1);
9702         if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
9703                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
9704         }
9705         else {
9706                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9707                 tw32(NVRAM_CFG1, nvcfg1);
9708         }
9709
9710         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) ||
9711             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
9712                 switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
9713                         case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
9714                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9715                                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
9716                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9717                                 break;
9718                         case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
9719                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9720                                 tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
9721                                 break;
9722                         case FLASH_VENDOR_ATMEL_EEPROM:
9723                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9724                                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9725                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9726                                 break;
9727                         case FLASH_VENDOR_ST:
9728                                 tp->nvram_jedecnum = JEDEC_ST;
9729                                 tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
9730                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9731                                 break;
9732                         case FLASH_VENDOR_SAIFUN:
9733                                 tp->nvram_jedecnum = JEDEC_SAIFUN;
9734                                 tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
9735                                 break;
9736                         case FLASH_VENDOR_SST_SMALL:
9737                         case FLASH_VENDOR_SST_LARGE:
9738                                 tp->nvram_jedecnum = JEDEC_SST;
9739                                 tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
9740                                 break;
9741                 }
9742         }
9743         else {
9744                 tp->nvram_jedecnum = JEDEC_ATMEL;
9745                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
9746                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9747         }
9748 }
9749
9750 static void __devinit tg3_get_5752_nvram_info(struct tg3 *tp)
9751 {
9752         u32 nvcfg1;
9753
9754         nvcfg1 = tr32(NVRAM_CFG1);
9755
9756         /* NVRAM protection for TPM */
9757         if (nvcfg1 & (1 << 27))
9758                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9759
9760         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9761                 case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
9762                 case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
9763                         tp->nvram_jedecnum = JEDEC_ATMEL;
9764                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9765                         break;
9766                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9767                         tp->nvram_jedecnum = JEDEC_ATMEL;
9768                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9769                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9770                         break;
9771                 case FLASH_5752VENDOR_ST_M45PE10:
9772                 case FLASH_5752VENDOR_ST_M45PE20:
9773                 case FLASH_5752VENDOR_ST_M45PE40:
9774                         tp->nvram_jedecnum = JEDEC_ST;
9775                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9776                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9777                         break;
9778         }
9779
9780         if (tp->tg3_flags2 & TG3_FLG2_FLASH) {
9781                 switch (nvcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
9782                         case FLASH_5752PAGE_SIZE_256:
9783                                 tp->nvram_pagesize = 256;
9784                                 break;
9785                         case FLASH_5752PAGE_SIZE_512:
9786                                 tp->nvram_pagesize = 512;
9787                                 break;
9788                         case FLASH_5752PAGE_SIZE_1K:
9789                                 tp->nvram_pagesize = 1024;
9790                                 break;
9791                         case FLASH_5752PAGE_SIZE_2K:
9792                                 tp->nvram_pagesize = 2048;
9793                                 break;
9794                         case FLASH_5752PAGE_SIZE_4K:
9795                                 tp->nvram_pagesize = 4096;
9796                                 break;
9797                         case FLASH_5752PAGE_SIZE_264:
9798                                 tp->nvram_pagesize = 264;
9799                                 break;
9800                 }
9801         }
9802         else {
9803                 /* For eeprom, set pagesize to maximum eeprom size */
9804                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9805
9806                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9807                 tw32(NVRAM_CFG1, nvcfg1);
9808         }
9809 }
9810
9811 static void __devinit tg3_get_5755_nvram_info(struct tg3 *tp)
9812 {
9813         u32 nvcfg1, protect = 0;
9814
9815         nvcfg1 = tr32(NVRAM_CFG1);
9816
9817         /* NVRAM protection for TPM */
9818         if (nvcfg1 & (1 << 27)) {
9819                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9820                 protect = 1;
9821         }
9822
9823         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
9824         switch (nvcfg1) {
9825                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
9826                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
9827                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
9828                 case FLASH_5755VENDOR_ATMEL_FLASH_5:
9829                         tp->nvram_jedecnum = JEDEC_ATMEL;
9830                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9831                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9832                         tp->nvram_pagesize = 264;
9833                         if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_1 ||
9834                             nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_5)
9835                                 tp->nvram_size = (protect ? 0x3e200 : 0x80000);
9836                         else if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_2)
9837                                 tp->nvram_size = (protect ? 0x1f200 : 0x40000);
9838                         else
9839                                 tp->nvram_size = (protect ? 0x1f200 : 0x20000);
9840                         break;
9841                 case FLASH_5752VENDOR_ST_M45PE10:
9842                 case FLASH_5752VENDOR_ST_M45PE20:
9843                 case FLASH_5752VENDOR_ST_M45PE40:
9844                         tp->nvram_jedecnum = JEDEC_ST;
9845                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9846                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9847                         tp->nvram_pagesize = 256;
9848                         if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE10)
9849                                 tp->nvram_size = (protect ? 0x10000 : 0x20000);
9850                         else if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE20)
9851                                 tp->nvram_size = (protect ? 0x10000 : 0x40000);
9852                         else
9853                                 tp->nvram_size = (protect ? 0x20000 : 0x80000);
9854                         break;
9855         }
9856 }
9857
9858 static void __devinit tg3_get_5787_nvram_info(struct tg3 *tp)
9859 {
9860         u32 nvcfg1;
9861
9862         nvcfg1 = tr32(NVRAM_CFG1);
9863
9864         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9865                 case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
9866                 case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
9867                 case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
9868                 case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
9869                         tp->nvram_jedecnum = JEDEC_ATMEL;
9870                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9871                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9872
9873                         nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9874                         tw32(NVRAM_CFG1, nvcfg1);
9875                         break;
9876                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9877                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
9878                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
9879                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
9880                         tp->nvram_jedecnum = JEDEC_ATMEL;
9881                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9882                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9883                         tp->nvram_pagesize = 264;
9884                         break;
9885                 case FLASH_5752VENDOR_ST_M45PE10:
9886                 case FLASH_5752VENDOR_ST_M45PE20:
9887                 case FLASH_5752VENDOR_ST_M45PE40:
9888                         tp->nvram_jedecnum = JEDEC_ST;
9889                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9890                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9891                         tp->nvram_pagesize = 256;
9892                         break;
9893         }
9894 }
9895
9896 static void __devinit tg3_get_5761_nvram_info(struct tg3 *tp)
9897 {
9898         u32 nvcfg1, protect = 0;
9899
9900         nvcfg1 = tr32(NVRAM_CFG1);
9901
9902         /* NVRAM protection for TPM */
9903         if (nvcfg1 & (1 << 27)) {
9904                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9905                 protect = 1;
9906         }
9907
9908         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
9909         switch (nvcfg1) {
9910                 case FLASH_5761VENDOR_ATMEL_ADB021D:
9911                 case FLASH_5761VENDOR_ATMEL_ADB041D:
9912                 case FLASH_5761VENDOR_ATMEL_ADB081D:
9913                 case FLASH_5761VENDOR_ATMEL_ADB161D:
9914                 case FLASH_5761VENDOR_ATMEL_MDB021D:
9915                 case FLASH_5761VENDOR_ATMEL_MDB041D:
9916                 case FLASH_5761VENDOR_ATMEL_MDB081D:
9917                 case FLASH_5761VENDOR_ATMEL_MDB161D:
9918                         tp->nvram_jedecnum = JEDEC_ATMEL;
9919                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9920                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9921                         tp->tg3_flags3 |= TG3_FLG3_NO_NVRAM_ADDR_TRANS;
9922                         tp->nvram_pagesize = 256;
9923                         break;
9924                 case FLASH_5761VENDOR_ST_A_M45PE20:
9925                 case FLASH_5761VENDOR_ST_A_M45PE40:
9926                 case FLASH_5761VENDOR_ST_A_M45PE80:
9927                 case FLASH_5761VENDOR_ST_A_M45PE16:
9928                 case FLASH_5761VENDOR_ST_M_M45PE20:
9929                 case FLASH_5761VENDOR_ST_M_M45PE40:
9930                 case FLASH_5761VENDOR_ST_M_M45PE80:
9931                 case FLASH_5761VENDOR_ST_M_M45PE16:
9932                         tp->nvram_jedecnum = JEDEC_ST;
9933                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9934                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9935                         tp->nvram_pagesize = 256;
9936                         break;
9937         }
9938
9939         if (protect) {
9940                 tp->nvram_size = tr32(NVRAM_ADDR_LOCKOUT);
9941         } else {
9942                 switch (nvcfg1) {
9943                         case FLASH_5761VENDOR_ATMEL_ADB161D:
9944                         case FLASH_5761VENDOR_ATMEL_MDB161D:
9945                         case FLASH_5761VENDOR_ST_A_M45PE16:
9946                         case FLASH_5761VENDOR_ST_M_M45PE16:
9947                                 tp->nvram_size = 0x100000;
9948                                 break;
9949                         case FLASH_5761VENDOR_ATMEL_ADB081D:
9950                         case FLASH_5761VENDOR_ATMEL_MDB081D:
9951                         case FLASH_5761VENDOR_ST_A_M45PE80:
9952                         case FLASH_5761VENDOR_ST_M_M45PE80:
9953                                 tp->nvram_size = 0x80000;
9954                                 break;
9955                         case FLASH_5761VENDOR_ATMEL_ADB041D:
9956                         case FLASH_5761VENDOR_ATMEL_MDB041D:
9957                         case FLASH_5761VENDOR_ST_A_M45PE40:
9958                         case FLASH_5761VENDOR_ST_M_M45PE40:
9959                                 tp->nvram_size = 0x40000;
9960                                 break;
9961                         case FLASH_5761VENDOR_ATMEL_ADB021D:
9962                         case FLASH_5761VENDOR_ATMEL_MDB021D:
9963                         case FLASH_5761VENDOR_ST_A_M45PE20:
9964                         case FLASH_5761VENDOR_ST_M_M45PE20:
9965                                 tp->nvram_size = 0x20000;
9966                                 break;
9967                 }
9968         }
9969 }
9970
9971 static void __devinit tg3_get_5906_nvram_info(struct tg3 *tp)
9972 {
9973         tp->nvram_jedecnum = JEDEC_ATMEL;
9974         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9975         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9976 }
9977
9978 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
9979 static void __devinit tg3_nvram_init(struct tg3 *tp)
9980 {
9981         tw32_f(GRC_EEPROM_ADDR,
9982              (EEPROM_ADDR_FSM_RESET |
9983               (EEPROM_DEFAULT_CLOCK_PERIOD <<
9984                EEPROM_ADDR_CLKPERD_SHIFT)));
9985
9986         msleep(1);
9987
9988         /* Enable seeprom accesses. */
9989         tw32_f(GRC_LOCAL_CTRL,
9990              tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
9991         udelay(100);
9992
9993         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
9994             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
9995                 tp->tg3_flags |= TG3_FLAG_NVRAM;
9996
9997                 if (tg3_nvram_lock(tp)) {
9998                         printk(KERN_WARNING PFX "%s: Cannot get nvarm lock, "
9999                                "tg3_nvram_init failed.\n", tp->dev->name);
10000                         return;
10001                 }
10002                 tg3_enable_nvram_access(tp);
10003
10004                 tp->nvram_size = 0;
10005
10006                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
10007                         tg3_get_5752_nvram_info(tp);
10008                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
10009                         tg3_get_5755_nvram_info(tp);
10010                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
10011                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784)
10012                         tg3_get_5787_nvram_info(tp);
10013                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
10014                         tg3_get_5761_nvram_info(tp);
10015                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
10016                         tg3_get_5906_nvram_info(tp);
10017                 else
10018                         tg3_get_nvram_info(tp);
10019
10020                 if (tp->nvram_size == 0)
10021                         tg3_get_nvram_size(tp);
10022
10023                 tg3_disable_nvram_access(tp);
10024                 tg3_nvram_unlock(tp);
10025
10026         } else {
10027                 tp->tg3_flags &= ~(TG3_FLAG_NVRAM | TG3_FLAG_NVRAM_BUFFERED);
10028
10029                 tg3_get_eeprom_size(tp);
10030         }
10031 }
10032
10033 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
10034                                         u32 offset, u32 *val)
10035 {
10036         u32 tmp;
10037         int i;
10038
10039         if (offset > EEPROM_ADDR_ADDR_MASK ||
10040             (offset % 4) != 0)
10041                 return -EINVAL;
10042
10043         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
10044                                         EEPROM_ADDR_DEVID_MASK |
10045                                         EEPROM_ADDR_READ);
10046         tw32(GRC_EEPROM_ADDR,
10047              tmp |
10048              (0 << EEPROM_ADDR_DEVID_SHIFT) |
10049              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
10050               EEPROM_ADDR_ADDR_MASK) |
10051              EEPROM_ADDR_READ | EEPROM_ADDR_START);
10052
10053         for (i = 0; i < 1000; i++) {
10054                 tmp = tr32(GRC_EEPROM_ADDR);
10055
10056                 if (tmp & EEPROM_ADDR_COMPLETE)
10057                         break;
10058                 msleep(1);
10059         }
10060         if (!(tmp & EEPROM_ADDR_COMPLETE))
10061                 return -EBUSY;
10062
10063         *val = tr32(GRC_EEPROM_DATA);
10064         return 0;
10065 }
10066
10067 #define NVRAM_CMD_TIMEOUT 10000
10068
10069 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
10070 {
10071         int i;
10072
10073         tw32(NVRAM_CMD, nvram_cmd);
10074         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
10075                 udelay(10);
10076                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
10077                         udelay(10);
10078                         break;
10079                 }
10080         }
10081         if (i == NVRAM_CMD_TIMEOUT) {
10082                 return -EBUSY;
10083         }
10084         return 0;
10085 }
10086
10087 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
10088 {
10089         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
10090             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
10091             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
10092            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
10093             (tp->nvram_jedecnum == JEDEC_ATMEL))
10094
10095                 addr = ((addr / tp->nvram_pagesize) <<
10096                         ATMEL_AT45DB0X1B_PAGE_POS) +
10097                        (addr % tp->nvram_pagesize);
10098
10099         return addr;
10100 }
10101
10102 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
10103 {
10104         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
10105             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
10106             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
10107            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
10108             (tp->nvram_jedecnum == JEDEC_ATMEL))
10109
10110                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
10111                         tp->nvram_pagesize) +
10112                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
10113
10114         return addr;
10115 }
10116
10117 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
10118 {
10119         int ret;
10120
10121         if (!(tp->tg3_flags & TG3_FLAG_NVRAM))
10122                 return tg3_nvram_read_using_eeprom(tp, offset, val);
10123
10124         offset = tg3_nvram_phys_addr(tp, offset);
10125
10126         if (offset > NVRAM_ADDR_MSK)
10127                 return -EINVAL;
10128
10129         ret = tg3_nvram_lock(tp);
10130         if (ret)
10131                 return ret;
10132
10133         tg3_enable_nvram_access(tp);
10134
10135         tw32(NVRAM_ADDR, offset);
10136         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
10137                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
10138
10139         if (ret == 0)
10140                 *val = swab32(tr32(NVRAM_RDDATA));
10141
10142         tg3_disable_nvram_access(tp);
10143
10144         tg3_nvram_unlock(tp);
10145
10146         return ret;
10147 }
10148
10149 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val)
10150 {
10151         int err;
10152         u32 tmp;
10153
10154         err = tg3_nvram_read(tp, offset, &tmp);
10155         *val = swab32(tmp);
10156         return err;
10157 }
10158
10159 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
10160                                     u32 offset, u32 len, u8 *buf)
10161 {
10162         int i, j, rc = 0;
10163         u32 val;
10164
10165         for (i = 0; i < len; i += 4) {
10166                 u32 addr, data;
10167
10168                 addr = offset + i;
10169
10170                 memcpy(&data, buf + i, 4);
10171
10172                 tw32(GRC_EEPROM_DATA, cpu_to_le32(data));
10173
10174                 val = tr32(GRC_EEPROM_ADDR);
10175                 tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
10176
10177                 val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
10178                         EEPROM_ADDR_READ);
10179                 tw32(GRC_EEPROM_ADDR, val |
10180                         (0 << EEPROM_ADDR_DEVID_SHIFT) |
10181                         (addr & EEPROM_ADDR_ADDR_MASK) |
10182                         EEPROM_ADDR_START |
10183                         EEPROM_ADDR_WRITE);
10184
10185                 for (j = 0; j < 1000; j++) {
10186                         val = tr32(GRC_EEPROM_ADDR);
10187
10188                         if (val & EEPROM_ADDR_COMPLETE)
10189                                 break;
10190                         msleep(1);
10191                 }
10192                 if (!(val & EEPROM_ADDR_COMPLETE)) {
10193                         rc = -EBUSY;
10194                         break;
10195                 }
10196         }
10197
10198         return rc;
10199 }
10200
10201 /* offset and length are dword aligned */
10202 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
10203                 u8 *buf)
10204 {
10205         int ret = 0;
10206         u32 pagesize = tp->nvram_pagesize;
10207         u32 pagemask = pagesize - 1;
10208         u32 nvram_cmd;
10209         u8 *tmp;
10210
10211         tmp = kmalloc(pagesize, GFP_KERNEL);
10212         if (tmp == NULL)
10213                 return -ENOMEM;
10214
10215         while (len) {
10216                 int j;
10217                 u32 phy_addr, page_off, size;
10218
10219                 phy_addr = offset & ~pagemask;
10220
10221                 for (j = 0; j < pagesize; j += 4) {
10222                         if ((ret = tg3_nvram_read(tp, phy_addr + j,
10223                                                 (u32 *) (tmp + j))))
10224                                 break;
10225                 }
10226                 if (ret)
10227                         break;
10228
10229                 page_off = offset & pagemask;
10230                 size = pagesize;
10231                 if (len < size)
10232                         size = len;
10233
10234                 len -= size;
10235
10236                 memcpy(tmp + page_off, buf, size);
10237
10238                 offset = offset + (pagesize - page_off);
10239
10240                 tg3_enable_nvram_access(tp);
10241
10242                 /*
10243                  * Before we can erase the flash page, we need
10244                  * to issue a special "write enable" command.
10245                  */
10246                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10247
10248                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10249                         break;
10250
10251                 /* Erase the target page */
10252                 tw32(NVRAM_ADDR, phy_addr);
10253
10254                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
10255                         NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
10256
10257                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10258                         break;
10259
10260                 /* Issue another write enable to start the write. */
10261                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10262
10263                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10264                         break;
10265
10266                 for (j = 0; j < pagesize; j += 4) {
10267                         u32 data;
10268
10269                         data = *((u32 *) (tmp + j));
10270                         tw32(NVRAM_WRDATA, cpu_to_be32(data));
10271
10272                         tw32(NVRAM_ADDR, phy_addr + j);
10273
10274                         nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
10275                                 NVRAM_CMD_WR;
10276
10277                         if (j == 0)
10278                                 nvram_cmd |= NVRAM_CMD_FIRST;
10279                         else if (j == (pagesize - 4))
10280                                 nvram_cmd |= NVRAM_CMD_LAST;
10281
10282                         if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
10283                                 break;
10284                 }
10285                 if (ret)
10286                         break;
10287         }
10288
10289         nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10290         tg3_nvram_exec_cmd(tp, nvram_cmd);
10291
10292         kfree(tmp);
10293
10294         return ret;
10295 }
10296
10297 /* offset and length are dword aligned */
10298 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
10299                 u8 *buf)
10300 {
10301         int i, ret = 0;
10302
10303         for (i = 0; i < len; i += 4, offset += 4) {
10304                 u32 data, page_off, phy_addr, nvram_cmd;
10305
10306                 memcpy(&data, buf + i, 4);
10307                 tw32(NVRAM_WRDATA, cpu_to_be32(data));
10308
10309                 page_off = offset % tp->nvram_pagesize;
10310
10311                 phy_addr = tg3_nvram_phys_addr(tp, offset);
10312
10313                 tw32(NVRAM_ADDR, phy_addr);
10314
10315                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
10316
10317                 if ((page_off == 0) || (i == 0))
10318                         nvram_cmd |= NVRAM_CMD_FIRST;
10319                 if (page_off == (tp->nvram_pagesize - 4))
10320                         nvram_cmd |= NVRAM_CMD_LAST;
10321
10322                 if (i == (len - 4))
10323                         nvram_cmd |= NVRAM_CMD_LAST;
10324
10325                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752) &&
10326                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755) &&
10327                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787) &&
10328                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784) &&
10329                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761) &&
10330                     (tp->nvram_jedecnum == JEDEC_ST) &&
10331                     (nvram_cmd & NVRAM_CMD_FIRST)) {
10332
10333                         if ((ret = tg3_nvram_exec_cmd(tp,
10334                                 NVRAM_CMD_WREN | NVRAM_CMD_GO |
10335                                 NVRAM_CMD_DONE)))
10336
10337                                 break;
10338                 }
10339                 if (!(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
10340                         /* We always do complete word writes to eeprom. */
10341                         nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
10342                 }
10343
10344                 if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
10345                         break;
10346         }
10347         return ret;
10348 }
10349
10350 /* offset and length are dword aligned */
10351 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
10352 {
10353         int ret;
10354
10355         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
10356                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
10357                        ~GRC_LCLCTRL_GPIO_OUTPUT1);
10358                 udelay(40);
10359         }
10360
10361         if (!(tp->tg3_flags & TG3_FLAG_NVRAM)) {
10362                 ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
10363         }
10364         else {
10365                 u32 grc_mode;
10366
10367                 ret = tg3_nvram_lock(tp);
10368                 if (ret)
10369                         return ret;
10370
10371                 tg3_enable_nvram_access(tp);
10372                 if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
10373                     !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM))
10374                         tw32(NVRAM_WRITE1, 0x406);
10375
10376                 grc_mode = tr32(GRC_MODE);
10377                 tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
10378
10379                 if ((tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) ||
10380                         !(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
10381
10382                         ret = tg3_nvram_write_block_buffered(tp, offset, len,
10383                                 buf);
10384                 }
10385                 else {
10386                         ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
10387                                 buf);
10388                 }
10389
10390                 grc_mode = tr32(GRC_MODE);
10391                 tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
10392
10393                 tg3_disable_nvram_access(tp);
10394                 tg3_nvram_unlock(tp);
10395         }
10396
10397         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
10398                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10399                 udelay(40);
10400         }
10401
10402         return ret;
10403 }
10404
10405 struct subsys_tbl_ent {
10406         u16 subsys_vendor, subsys_devid;
10407         u32 phy_id;
10408 };
10409
10410 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
10411         /* Broadcom boards. */
10412         { PCI_VENDOR_ID_BROADCOM, 0x1644, PHY_ID_BCM5401 }, /* BCM95700A6 */
10413         { PCI_VENDOR_ID_BROADCOM, 0x0001, PHY_ID_BCM5701 }, /* BCM95701A5 */
10414         { PCI_VENDOR_ID_BROADCOM, 0x0002, PHY_ID_BCM8002 }, /* BCM95700T6 */
10415         { PCI_VENDOR_ID_BROADCOM, 0x0003, 0 },              /* BCM95700A9 */
10416         { PCI_VENDOR_ID_BROADCOM, 0x0005, PHY_ID_BCM5701 }, /* BCM95701T1 */
10417         { PCI_VENDOR_ID_BROADCOM, 0x0006, PHY_ID_BCM5701 }, /* BCM95701T8 */
10418         { PCI_VENDOR_ID_BROADCOM, 0x0007, 0 },              /* BCM95701A7 */
10419         { PCI_VENDOR_ID_BROADCOM, 0x0008, PHY_ID_BCM5701 }, /* BCM95701A10 */
10420         { PCI_VENDOR_ID_BROADCOM, 0x8008, PHY_ID_BCM5701 }, /* BCM95701A12 */
10421         { PCI_VENDOR_ID_BROADCOM, 0x0009, PHY_ID_BCM5703 }, /* BCM95703Ax1 */
10422         { PCI_VENDOR_ID_BROADCOM, 0x8009, PHY_ID_BCM5703 }, /* BCM95703Ax2 */
10423
10424         /* 3com boards. */
10425         { PCI_VENDOR_ID_3COM, 0x1000, PHY_ID_BCM5401 }, /* 3C996T */
10426         { PCI_VENDOR_ID_3COM, 0x1006, PHY_ID_BCM5701 }, /* 3C996BT */
10427         { PCI_VENDOR_ID_3COM, 0x1004, 0 },              /* 3C996SX */
10428         { PCI_VENDOR_ID_3COM, 0x1007, PHY_ID_BCM5701 }, /* 3C1000T */
10429         { PCI_VENDOR_ID_3COM, 0x1008, PHY_ID_BCM5701 }, /* 3C940BR01 */
10430
10431         /* DELL boards. */
10432         { PCI_VENDOR_ID_DELL, 0x00d1, PHY_ID_BCM5401 }, /* VIPER */
10433         { PCI_VENDOR_ID_DELL, 0x0106, PHY_ID_BCM5401 }, /* JAGUAR */
10434         { PCI_VENDOR_ID_DELL, 0x0109, PHY_ID_BCM5411 }, /* MERLOT */
10435         { PCI_VENDOR_ID_DELL, 0x010a, PHY_ID_BCM5411 }, /* SLIM_MERLOT */
10436
10437         /* Compaq boards. */
10438         { PCI_VENDOR_ID_COMPAQ, 0x007c, PHY_ID_BCM5701 }, /* BANSHEE */
10439         { PCI_VENDOR_ID_COMPAQ, 0x009a, PHY_ID_BCM5701 }, /* BANSHEE_2 */
10440         { PCI_VENDOR_ID_COMPAQ, 0x007d, 0 },              /* CHANGELING */
10441         { PCI_VENDOR_ID_COMPAQ, 0x0085, PHY_ID_BCM5701 }, /* NC7780 */
10442         { PCI_VENDOR_ID_COMPAQ, 0x0099, PHY_ID_BCM5701 }, /* NC7780_2 */
10443
10444         /* IBM boards. */
10445         { PCI_VENDOR_ID_IBM, 0x0281, 0 } /* IBM??? */
10446 };
10447
10448 static inline struct subsys_tbl_ent *lookup_by_subsys(struct tg3 *tp)
10449 {
10450         int i;
10451
10452         for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
10453                 if ((subsys_id_to_phy_id[i].subsys_vendor ==
10454                      tp->pdev->subsystem_vendor) &&
10455                     (subsys_id_to_phy_id[i].subsys_devid ==
10456                      tp->pdev->subsystem_device))
10457                         return &subsys_id_to_phy_id[i];
10458         }
10459         return NULL;
10460 }
10461
10462 static void __devinit tg3_get_eeprom_hw_cfg(struct tg3 *tp)
10463 {
10464         u32 val;
10465         u16 pmcsr;
10466
10467         /* On some early chips the SRAM cannot be accessed in D3hot state,
10468          * so need make sure we're in D0.
10469          */
10470         pci_read_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, &pmcsr);
10471         pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
10472         pci_write_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, pmcsr);
10473         msleep(1);
10474
10475         /* Make sure register accesses (indirect or otherwise)
10476          * will function correctly.
10477          */
10478         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10479                                tp->misc_host_ctrl);
10480
10481         /* The memory arbiter has to be enabled in order for SRAM accesses
10482          * to succeed.  Normally on powerup the tg3 chip firmware will make
10483          * sure it is enabled, but other entities such as system netboot
10484          * code might disable it.
10485          */
10486         val = tr32(MEMARB_MODE);
10487         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
10488
10489         tp->phy_id = PHY_ID_INVALID;
10490         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10491
10492         /* Assume an onboard device and WOL capable by default.  */
10493         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT | TG3_FLAG_WOL_CAP;
10494
10495         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
10496                 if (!(tr32(PCIE_TRANSACTION_CFG) & PCIE_TRANS_CFG_LOM)) {
10497                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10498                         tp->tg3_flags2 |= TG3_FLG2_IS_NIC;
10499                 }
10500                 val = tr32(VCPU_CFGSHDW);
10501                 if (val & VCPU_CFGSHDW_ASPM_DBNC)
10502                         tp->tg3_flags |= TG3_FLAG_ASPM_WORKAROUND;
10503                 if ((val & VCPU_CFGSHDW_WOL_ENABLE) &&
10504                     (val & VCPU_CFGSHDW_WOL_MAGPKT))
10505                         tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
10506                 return;
10507         }
10508
10509         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
10510         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
10511                 u32 nic_cfg, led_cfg;
10512                 u32 nic_phy_id, ver, cfg2 = 0, eeprom_phy_id;
10513                 int eeprom_phy_serdes = 0;
10514
10515                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
10516                 tp->nic_sram_data_cfg = nic_cfg;
10517
10518                 tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
10519                 ver >>= NIC_SRAM_DATA_VER_SHIFT;
10520                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700) &&
10521                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) &&
10522                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5703) &&
10523                     (ver > 0) && (ver < 0x100))
10524                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
10525
10526                 if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
10527                     NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
10528                         eeprom_phy_serdes = 1;
10529
10530                 tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
10531                 if (nic_phy_id != 0) {
10532                         u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
10533                         u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
10534
10535                         eeprom_phy_id  = (id1 >> 16) << 10;
10536                         eeprom_phy_id |= (id2 & 0xfc00) << 16;
10537                         eeprom_phy_id |= (id2 & 0x03ff) <<  0;
10538                 } else
10539                         eeprom_phy_id = 0;
10540
10541                 tp->phy_id = eeprom_phy_id;
10542                 if (eeprom_phy_serdes) {
10543                         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
10544                                 tp->tg3_flags2 |= TG3_FLG2_MII_SERDES;
10545                         else
10546                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10547                 }
10548
10549                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
10550                         led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
10551                                     SHASTA_EXT_LED_MODE_MASK);
10552                 else
10553                         led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
10554
10555                 switch (led_cfg) {
10556                 default:
10557                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
10558                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10559                         break;
10560
10561                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
10562                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
10563                         break;
10564
10565                 case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
10566                         tp->led_ctrl = LED_CTRL_MODE_MAC;
10567
10568                         /* Default to PHY_1_MODE if 0 (MAC_MODE) is
10569                          * read on some older 5700/5701 bootcode.
10570                          */
10571                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
10572                             ASIC_REV_5700 ||
10573                             GET_ASIC_REV(tp->pci_chip_rev_id) ==
10574                             ASIC_REV_5701)
10575                                 tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10576
10577                         break;
10578
10579                 case SHASTA_EXT_LED_SHARED:
10580                         tp->led_ctrl = LED_CTRL_MODE_SHARED;
10581                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
10582                             tp->pci_chip_rev_id != CHIPREV_ID_5750_A1)
10583                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
10584                                                  LED_CTRL_MODE_PHY_2);
10585                         break;
10586
10587                 case SHASTA_EXT_LED_MAC:
10588                         tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
10589                         break;
10590
10591                 case SHASTA_EXT_LED_COMBO:
10592                         tp->led_ctrl = LED_CTRL_MODE_COMBO;
10593                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0)
10594                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
10595                                                  LED_CTRL_MODE_PHY_2);
10596                         break;
10597
10598                 };
10599
10600                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
10601                      GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) &&
10602                     tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
10603                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
10604
10605                 if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP) {
10606                         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT;
10607                         if ((tp->pdev->subsystem_vendor ==
10608                              PCI_VENDOR_ID_ARIMA) &&
10609                             (tp->pdev->subsystem_device == 0x205a ||
10610                              tp->pdev->subsystem_device == 0x2063))
10611                                 tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10612                 } else {
10613                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10614                         tp->tg3_flags2 |= TG3_FLG2_IS_NIC;
10615                 }
10616
10617                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
10618                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
10619                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
10620                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
10621                 }
10622                 if (nic_cfg & NIC_SRAM_DATA_CFG_APE_ENABLE)
10623                         tp->tg3_flags3 |= TG3_FLG3_ENABLE_APE;
10624                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES &&
10625                     !(nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL))
10626                         tp->tg3_flags &= ~TG3_FLAG_WOL_CAP;
10627
10628                 if (tp->tg3_flags & TG3_FLAG_WOL_CAP &&
10629                     nic_cfg & NIC_SRAM_DATA_CFG_WOL_ENABLE)
10630                         tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
10631
10632                 if (cfg2 & (1 << 17))
10633                         tp->tg3_flags2 |= TG3_FLG2_CAPACITIVE_COUPLING;
10634
10635                 /* serdes signal pre-emphasis in register 0x590 set by */
10636                 /* bootcode if bit 18 is set */
10637                 if (cfg2 & (1 << 18))
10638                         tp->tg3_flags2 |= TG3_FLG2_SERDES_PREEMPHASIS;
10639
10640                 if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
10641                         u32 cfg3;
10642
10643                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &cfg3);
10644                         if (cfg3 & NIC_SRAM_ASPM_DEBOUNCE)
10645                                 tp->tg3_flags |= TG3_FLAG_ASPM_WORKAROUND;
10646                 }
10647         }
10648 }
10649
10650 static int __devinit tg3_phy_probe(struct tg3 *tp)
10651 {
10652         u32 hw_phy_id_1, hw_phy_id_2;
10653         u32 hw_phy_id, hw_phy_id_masked;
10654         int err;
10655
10656         /* Reading the PHY ID register can conflict with ASF
10657          * firwmare access to the PHY hardware.
10658          */
10659         err = 0;
10660         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
10661             (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
10662                 hw_phy_id = hw_phy_id_masked = PHY_ID_INVALID;
10663         } else {
10664                 /* Now read the physical PHY_ID from the chip and verify
10665                  * that it is sane.  If it doesn't look good, we fall back
10666                  * to either the hard-coded table based PHY_ID and failing
10667                  * that the value found in the eeprom area.
10668                  */
10669                 err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
10670                 err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
10671
10672                 hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
10673                 hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
10674                 hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
10675
10676                 hw_phy_id_masked = hw_phy_id & PHY_ID_MASK;
10677         }
10678
10679         if (!err && KNOWN_PHY_ID(hw_phy_id_masked)) {
10680                 tp->phy_id = hw_phy_id;
10681                 if (hw_phy_id_masked == PHY_ID_BCM8002)
10682                         tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10683                 else
10684                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_SERDES;
10685         } else {
10686                 if (tp->phy_id != PHY_ID_INVALID) {
10687                         /* Do nothing, phy ID already set up in
10688                          * tg3_get_eeprom_hw_cfg().
10689                          */
10690                 } else {
10691                         struct subsys_tbl_ent *p;
10692
10693                         /* No eeprom signature?  Try the hardcoded
10694                          * subsys device table.
10695                          */
10696                         p = lookup_by_subsys(tp);
10697                         if (!p)
10698                                 return -ENODEV;
10699
10700                         tp->phy_id = p->phy_id;
10701                         if (!tp->phy_id ||
10702                             tp->phy_id == PHY_ID_BCM8002)
10703                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10704                 }
10705         }
10706
10707         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) &&
10708             !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) &&
10709             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
10710                 u32 bmsr, adv_reg, tg3_ctrl, mask;
10711
10712                 tg3_readphy(tp, MII_BMSR, &bmsr);
10713                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
10714                     (bmsr & BMSR_LSTATUS))
10715                         goto skip_phy_reset;
10716
10717                 err = tg3_phy_reset(tp);
10718                 if (err)
10719                         return err;
10720
10721                 adv_reg = (ADVERTISE_10HALF | ADVERTISE_10FULL |
10722                            ADVERTISE_100HALF | ADVERTISE_100FULL |
10723                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
10724                 tg3_ctrl = 0;
10725                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
10726                         tg3_ctrl = (MII_TG3_CTRL_ADV_1000_HALF |
10727                                     MII_TG3_CTRL_ADV_1000_FULL);
10728                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
10729                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
10730                                 tg3_ctrl |= (MII_TG3_CTRL_AS_MASTER |
10731                                              MII_TG3_CTRL_ENABLE_AS_MASTER);
10732                 }
10733
10734                 mask = (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
10735                         ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
10736                         ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full);
10737                 if (!tg3_copper_is_advertising_all(tp, mask)) {
10738                         tg3_writephy(tp, MII_ADVERTISE, adv_reg);
10739
10740                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
10741                                 tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
10742
10743                         tg3_writephy(tp, MII_BMCR,
10744                                      BMCR_ANENABLE | BMCR_ANRESTART);
10745                 }
10746                 tg3_phy_set_wirespeed(tp);
10747
10748                 tg3_writephy(tp, MII_ADVERTISE, adv_reg);
10749                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
10750                         tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
10751         }
10752
10753 skip_phy_reset:
10754         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
10755                 err = tg3_init_5401phy_dsp(tp);
10756                 if (err)
10757                         return err;
10758         }
10759
10760         if (!err && ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)) {
10761                 err = tg3_init_5401phy_dsp(tp);
10762         }
10763
10764         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
10765                 tp->link_config.advertising =
10766                         (ADVERTISED_1000baseT_Half |
10767                          ADVERTISED_1000baseT_Full |
10768                          ADVERTISED_Autoneg |
10769                          ADVERTISED_FIBRE);
10770         if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
10771                 tp->link_config.advertising &=
10772                         ~(ADVERTISED_1000baseT_Half |
10773                           ADVERTISED_1000baseT_Full);
10774
10775         return err;
10776 }
10777
10778 static void __devinit tg3_read_partno(struct tg3 *tp)
10779 {
10780         unsigned char vpd_data[256];
10781         unsigned int i;
10782         u32 magic;
10783
10784         if (tg3_nvram_read_swab(tp, 0x0, &magic))
10785                 goto out_not_found;
10786
10787         if (magic == TG3_EEPROM_MAGIC) {
10788                 for (i = 0; i < 256; i += 4) {
10789                         u32 tmp;
10790
10791                         if (tg3_nvram_read(tp, 0x100 + i, &tmp))
10792                                 goto out_not_found;
10793
10794                         vpd_data[i + 0] = ((tmp >>  0) & 0xff);
10795                         vpd_data[i + 1] = ((tmp >>  8) & 0xff);
10796                         vpd_data[i + 2] = ((tmp >> 16) & 0xff);
10797                         vpd_data[i + 3] = ((tmp >> 24) & 0xff);
10798                 }
10799         } else {
10800                 int vpd_cap;
10801
10802                 vpd_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_VPD);
10803                 for (i = 0; i < 256; i += 4) {
10804                         u32 tmp, j = 0;
10805                         u16 tmp16;
10806
10807                         pci_write_config_word(tp->pdev, vpd_cap + PCI_VPD_ADDR,
10808                                               i);
10809                         while (j++ < 100) {
10810                                 pci_read_config_word(tp->pdev, vpd_cap +
10811                                                      PCI_VPD_ADDR, &tmp16);
10812                                 if (tmp16 & 0x8000)
10813                                         break;
10814                                 msleep(1);
10815                         }
10816                         if (!(tmp16 & 0x8000))
10817                                 goto out_not_found;
10818
10819                         pci_read_config_dword(tp->pdev, vpd_cap + PCI_VPD_DATA,
10820                                               &tmp);
10821                         tmp = cpu_to_le32(tmp);
10822                         memcpy(&vpd_data[i], &tmp, 4);
10823                 }
10824         }
10825
10826         /* Now parse and find the part number. */
10827         for (i = 0; i < 254; ) {
10828                 unsigned char val = vpd_data[i];
10829                 unsigned int block_end;
10830
10831                 if (val == 0x82 || val == 0x91) {
10832                         i = (i + 3 +
10833                              (vpd_data[i + 1] +
10834                               (vpd_data[i + 2] << 8)));
10835                         continue;
10836                 }
10837
10838                 if (val != 0x90)
10839                         goto out_not_found;
10840
10841                 block_end = (i + 3 +
10842                              (vpd_data[i + 1] +
10843                               (vpd_data[i + 2] << 8)));
10844                 i += 3;
10845
10846                 if (block_end > 256)
10847                         goto out_not_found;
10848
10849                 while (i < (block_end - 2)) {
10850                         if (vpd_data[i + 0] == 'P' &&
10851                             vpd_data[i + 1] == 'N') {
10852                                 int partno_len = vpd_data[i + 2];
10853
10854                                 i += 3;
10855                                 if (partno_len > 24 || (partno_len + i) > 256)
10856                                         goto out_not_found;
10857
10858                                 memcpy(tp->board_part_number,
10859                                        &vpd_data[i], partno_len);
10860
10861                                 /* Success. */
10862                                 return;
10863                         }
10864                         i += 3 + vpd_data[i + 2];
10865                 }
10866
10867                 /* Part number not found. */
10868                 goto out_not_found;
10869         }
10870
10871 out_not_found:
10872         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
10873                 strcpy(tp->board_part_number, "BCM95906");
10874         else
10875                 strcpy(tp->board_part_number, "none");
10876 }
10877
10878 static int __devinit tg3_fw_img_is_valid(struct tg3 *tp, u32 offset)
10879 {
10880         u32 val;
10881
10882         if (tg3_nvram_read_swab(tp, offset, &val) ||
10883             (val & 0xfc000000) != 0x0c000000 ||
10884             tg3_nvram_read_swab(tp, offset + 4, &val) ||
10885             val != 0)
10886                 return 0;
10887
10888         return 1;
10889 }
10890
10891 static void __devinit tg3_read_fw_ver(struct tg3 *tp)
10892 {
10893         u32 val, offset, start;
10894         u32 ver_offset;
10895         int i, bcnt;
10896
10897         if (tg3_nvram_read_swab(tp, 0, &val))
10898                 return;
10899
10900         if (val != TG3_EEPROM_MAGIC)
10901                 return;
10902
10903         if (tg3_nvram_read_swab(tp, 0xc, &offset) ||
10904             tg3_nvram_read_swab(tp, 0x4, &start))
10905                 return;
10906
10907         offset = tg3_nvram_logical_addr(tp, offset);
10908
10909         if (!tg3_fw_img_is_valid(tp, offset) ||
10910             tg3_nvram_read_swab(tp, offset + 8, &ver_offset))
10911                 return;
10912
10913         offset = offset + ver_offset - start;
10914         for (i = 0; i < 16; i += 4) {
10915                 if (tg3_nvram_read(tp, offset + i, &val))
10916                         return;
10917
10918                 val = le32_to_cpu(val);
10919                 memcpy(tp->fw_ver + i, &val, 4);
10920         }
10921
10922         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
10923              (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
10924                 return;
10925
10926         for (offset = TG3_NVM_DIR_START;
10927              offset < TG3_NVM_DIR_END;
10928              offset += TG3_NVM_DIRENT_SIZE) {
10929                 if (tg3_nvram_read_swab(tp, offset, &val))
10930                         return;
10931
10932                 if ((val >> TG3_NVM_DIRTYPE_SHIFT) == TG3_NVM_DIRTYPE_ASFINI)
10933                         break;
10934         }
10935
10936         if (offset == TG3_NVM_DIR_END)
10937                 return;
10938
10939         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
10940                 start = 0x08000000;
10941         else if (tg3_nvram_read_swab(tp, offset - 4, &start))
10942                 return;
10943
10944         if (tg3_nvram_read_swab(tp, offset + 4, &offset) ||
10945             !tg3_fw_img_is_valid(tp, offset) ||
10946             tg3_nvram_read_swab(tp, offset + 8, &val))
10947                 return;
10948
10949         offset += val - start;
10950
10951         bcnt = strlen(tp->fw_ver);
10952
10953         tp->fw_ver[bcnt++] = ',';
10954         tp->fw_ver[bcnt++] = ' ';
10955
10956         for (i = 0; i < 4; i++) {
10957                 if (tg3_nvram_read(tp, offset, &val))
10958                         return;
10959
10960                 val = le32_to_cpu(val);
10961                 offset += sizeof(val);
10962
10963                 if (bcnt > TG3_VER_SIZE - sizeof(val)) {
10964                         memcpy(&tp->fw_ver[bcnt], &val, TG3_VER_SIZE - bcnt);
10965                         break;
10966                 }
10967
10968                 memcpy(&tp->fw_ver[bcnt], &val, sizeof(val));
10969                 bcnt += sizeof(val);
10970         }
10971
10972         tp->fw_ver[TG3_VER_SIZE - 1] = 0;
10973 }
10974
10975 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *);
10976
10977 static int __devinit tg3_get_invariants(struct tg3 *tp)
10978 {
10979         static struct pci_device_id write_reorder_chipsets[] = {
10980                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
10981                              PCI_DEVICE_ID_AMD_FE_GATE_700C) },
10982                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
10983                              PCI_DEVICE_ID_AMD_8131_BRIDGE) },
10984                 { PCI_DEVICE(PCI_VENDOR_ID_VIA,
10985                              PCI_DEVICE_ID_VIA_8385_0) },
10986                 { },
10987         };
10988         u32 misc_ctrl_reg;
10989         u32 cacheline_sz_reg;
10990         u32 pci_state_reg, grc_misc_cfg;
10991         u32 val;
10992         u16 pci_cmd;
10993         int err, pcie_cap;
10994
10995         /* Force memory write invalidate off.  If we leave it on,
10996          * then on 5700_BX chips we have to enable a workaround.
10997          * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
10998          * to match the cacheline size.  The Broadcom driver have this
10999          * workaround but turns MWI off all the times so never uses
11000          * it.  This seems to suggest that the workaround is insufficient.
11001          */
11002         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11003         pci_cmd &= ~PCI_COMMAND_INVALIDATE;
11004         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11005
11006         /* It is absolutely critical that TG3PCI_MISC_HOST_CTRL
11007          * has the register indirect write enable bit set before
11008          * we try to access any of the MMIO registers.  It is also
11009          * critical that the PCI-X hw workaround situation is decided
11010          * before that as well.
11011          */
11012         pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11013                               &misc_ctrl_reg);
11014
11015         tp->pci_chip_rev_id = (misc_ctrl_reg >>
11016                                MISC_HOST_CTRL_CHIPREV_SHIFT);
11017         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_USE_PROD_ID_REG) {
11018                 u32 prod_id_asic_rev;
11019
11020                 pci_read_config_dword(tp->pdev, TG3PCI_PRODID_ASICREV,
11021                                       &prod_id_asic_rev);
11022                 tp->pci_chip_rev_id = prod_id_asic_rev & PROD_ID_ASIC_REV_MASK;
11023         }
11024
11025         /* Wrong chip ID in 5752 A0. This code can be removed later
11026          * as A0 is not in production.
11027          */
11028         if (tp->pci_chip_rev_id == CHIPREV_ID_5752_A0_HW)
11029                 tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
11030
11031         /* If we have 5702/03 A1 or A2 on certain ICH chipsets,
11032          * we need to disable memory and use config. cycles
11033          * only to access all registers. The 5702/03 chips
11034          * can mistakenly decode the special cycles from the
11035          * ICH chipsets as memory write cycles, causing corruption
11036          * of register and memory space. Only certain ICH bridges
11037          * will drive special cycles with non-zero data during the
11038          * address phase which can fall within the 5703's address
11039          * range. This is not an ICH bug as the PCI spec allows
11040          * non-zero address during special cycles. However, only
11041          * these ICH bridges are known to drive non-zero addresses
11042          * during special cycles.
11043          *
11044          * Since special cycles do not cross PCI bridges, we only
11045          * enable this workaround if the 5703 is on the secondary
11046          * bus of these ICH bridges.
11047          */
11048         if ((tp->pci_chip_rev_id == CHIPREV_ID_5703_A1) ||
11049             (tp->pci_chip_rev_id == CHIPREV_ID_5703_A2)) {
11050                 static struct tg3_dev_id {
11051                         u32     vendor;
11052                         u32     device;
11053                         u32     rev;
11054                 } ich_chipsets[] = {
11055                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
11056                           PCI_ANY_ID },
11057                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
11058                           PCI_ANY_ID },
11059                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
11060                           0xa },
11061                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
11062                           PCI_ANY_ID },
11063                         { },
11064                 };
11065                 struct tg3_dev_id *pci_id = &ich_chipsets[0];
11066                 struct pci_dev *bridge = NULL;
11067
11068                 while (pci_id->vendor != 0) {
11069                         bridge = pci_get_device(pci_id->vendor, pci_id->device,
11070                                                 bridge);
11071                         if (!bridge) {
11072                                 pci_id++;
11073                                 continue;
11074                         }
11075                         if (pci_id->rev != PCI_ANY_ID) {
11076                                 if (bridge->revision > pci_id->rev)
11077                                         continue;
11078                         }
11079                         if (bridge->subordinate &&
11080                             (bridge->subordinate->number ==
11081                              tp->pdev->bus->number)) {
11082
11083                                 tp->tg3_flags2 |= TG3_FLG2_ICH_WORKAROUND;
11084                                 pci_dev_put(bridge);
11085                                 break;
11086                         }
11087                 }
11088         }
11089
11090         /* The EPB bridge inside 5714, 5715, and 5780 cannot support
11091          * DMA addresses > 40-bit. This bridge may have other additional
11092          * 57xx devices behind it in some 4-port NIC designs for example.
11093          * Any tg3 device found behind the bridge will also need the 40-bit
11094          * DMA workaround.
11095          */
11096         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 ||
11097             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
11098                 tp->tg3_flags2 |= TG3_FLG2_5780_CLASS;
11099                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
11100                 tp->msi_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_MSI);
11101         }
11102         else {
11103                 struct pci_dev *bridge = NULL;
11104
11105                 do {
11106                         bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
11107                                                 PCI_DEVICE_ID_SERVERWORKS_EPB,
11108                                                 bridge);
11109                         if (bridge && bridge->subordinate &&
11110                             (bridge->subordinate->number <=
11111                              tp->pdev->bus->number) &&
11112                             (bridge->subordinate->subordinate >=
11113                              tp->pdev->bus->number)) {
11114                                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
11115                                 pci_dev_put(bridge);
11116                                 break;
11117                         }
11118                 } while (bridge);
11119         }
11120
11121         /* Initialize misc host control in PCI block. */
11122         tp->misc_host_ctrl |= (misc_ctrl_reg &
11123                                MISC_HOST_CTRL_CHIPREV);
11124         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11125                                tp->misc_host_ctrl);
11126
11127         pci_read_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
11128                               &cacheline_sz_reg);
11129
11130         tp->pci_cacheline_sz = (cacheline_sz_reg >>  0) & 0xff;
11131         tp->pci_lat_timer    = (cacheline_sz_reg >>  8) & 0xff;
11132         tp->pci_hdr_type     = (cacheline_sz_reg >> 16) & 0xff;
11133         tp->pci_bist         = (cacheline_sz_reg >> 24) & 0xff;
11134
11135         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
11136             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714))
11137                 tp->pdev_peer = tg3_find_peer(tp);
11138
11139         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
11140             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
11141             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11142             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11143             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11144             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11145             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
11146             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
11147                 tp->tg3_flags2 |= TG3_FLG2_5750_PLUS;
11148
11149         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) ||
11150             (tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
11151                 tp->tg3_flags2 |= TG3_FLG2_5705_PLUS;
11152
11153         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
11154                 tp->tg3_flags |= TG3_FLAG_SUPPORT_MSI;
11155                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX ||
11156                     GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX ||
11157                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 &&
11158                      tp->pci_chip_rev_id <= CHIPREV_ID_5714_A2 &&
11159                      tp->pdev_peer == tp->pdev))
11160                         tp->tg3_flags &= ~TG3_FLAG_SUPPORT_MSI;
11161
11162                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11163                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11164                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11165                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11166                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11167                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_2;
11168                         tp->tg3_flags2 |= TG3_FLG2_1SHOT_MSI;
11169                 } else {
11170                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_1 | TG3_FLG2_TSO_BUG;
11171                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
11172                                 ASIC_REV_5750 &&
11173                             tp->pci_chip_rev_id >= CHIPREV_ID_5750_C2)
11174                                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_BUG;
11175                 }
11176         }
11177
11178         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705 &&
11179             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5750 &&
11180             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752 &&
11181             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755 &&
11182             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787 &&
11183             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 &&
11184             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761 &&
11185             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
11186                 tp->tg3_flags2 |= TG3_FLG2_JUMBO_CAPABLE;
11187
11188         pcie_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_EXP);
11189         if (pcie_cap != 0) {
11190                 tp->tg3_flags2 |= TG3_FLG2_PCI_EXPRESS;
11191                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11192                         u16 lnkctl;
11193
11194                         pci_read_config_word(tp->pdev,
11195                                              pcie_cap + PCI_EXP_LNKCTL,
11196                                              &lnkctl);
11197                         if (lnkctl & PCI_EXP_LNKCTL_CLKREQ_EN)
11198                                 tp->tg3_flags2 &= ~TG3_FLG2_HW_TSO_2;
11199                 }
11200         }
11201
11202         /* If we have an AMD 762 or VIA K8T800 chipset, write
11203          * reordering to the mailbox registers done by the host
11204          * controller can cause major troubles.  We read back from
11205          * every mailbox register write to force the writes to be
11206          * posted to the chip in order.
11207          */
11208         if (pci_dev_present(write_reorder_chipsets) &&
11209             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
11210                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
11211
11212         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
11213             tp->pci_lat_timer < 64) {
11214                 tp->pci_lat_timer = 64;
11215
11216                 cacheline_sz_reg  = ((tp->pci_cacheline_sz & 0xff) <<  0);
11217                 cacheline_sz_reg |= ((tp->pci_lat_timer    & 0xff) <<  8);
11218                 cacheline_sz_reg |= ((tp->pci_hdr_type     & 0xff) << 16);
11219                 cacheline_sz_reg |= ((tp->pci_bist         & 0xff) << 24);
11220
11221                 pci_write_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
11222                                        cacheline_sz_reg);
11223         }
11224
11225         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
11226             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
11227                 tp->pcix_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_PCIX);
11228                 if (!tp->pcix_cap) {
11229                         printk(KERN_ERR PFX "Cannot find PCI-X "
11230                                             "capability, aborting.\n");
11231                         return -EIO;
11232                 }
11233         }
11234
11235         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
11236                               &pci_state_reg);
11237
11238         if (tp->pcix_cap && (pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0) {
11239                 tp->tg3_flags |= TG3_FLAG_PCIX_MODE;
11240
11241                 /* If this is a 5700 BX chipset, and we are in PCI-X
11242                  * mode, enable register write workaround.
11243                  *
11244                  * The workaround is to use indirect register accesses
11245                  * for all chip writes not to mailbox registers.
11246                  */
11247                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX) {
11248                         u32 pm_reg;
11249
11250                         tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
11251
11252                         /* The chip can have it's power management PCI config
11253                          * space registers clobbered due to this bug.
11254                          * So explicitly force the chip into D0 here.
11255                          */
11256                         pci_read_config_dword(tp->pdev,
11257                                               tp->pm_cap + PCI_PM_CTRL,
11258                                               &pm_reg);
11259                         pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
11260                         pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
11261                         pci_write_config_dword(tp->pdev,
11262                                                tp->pm_cap + PCI_PM_CTRL,
11263                                                pm_reg);
11264
11265                         /* Also, force SERR#/PERR# in PCI command. */
11266                         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11267                         pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
11268                         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11269                 }
11270         }
11271
11272         /* 5700 BX chips need to have their TX producer index mailboxes
11273          * written twice to workaround a bug.
11274          */
11275         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX)
11276                 tp->tg3_flags |= TG3_FLAG_TXD_MBOX_HWBUG;
11277
11278         if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
11279                 tp->tg3_flags |= TG3_FLAG_PCI_HIGH_SPEED;
11280         if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
11281                 tp->tg3_flags |= TG3_FLAG_PCI_32BIT;
11282
11283         /* Chip-specific fixup from Broadcom driver */
11284         if ((tp->pci_chip_rev_id == CHIPREV_ID_5704_A0) &&
11285             (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
11286                 pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
11287                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
11288         }
11289
11290         /* Default fast path register access methods */
11291         tp->read32 = tg3_read32;
11292         tp->write32 = tg3_write32;
11293         tp->read32_mbox = tg3_read32;
11294         tp->write32_mbox = tg3_write32;
11295         tp->write32_tx_mbox = tg3_write32;
11296         tp->write32_rx_mbox = tg3_write32;
11297
11298         /* Various workaround register access methods */
11299         if (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG)
11300                 tp->write32 = tg3_write_indirect_reg32;
11301         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
11302                  ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
11303                   tp->pci_chip_rev_id == CHIPREV_ID_5750_A0)) {
11304                 /*
11305                  * Back to back register writes can cause problems on these
11306                  * chips, the workaround is to read back all reg writes
11307                  * except those to mailbox regs.
11308                  *
11309                  * See tg3_write_indirect_reg32().
11310                  */
11311                 tp->write32 = tg3_write_flush_reg32;
11312         }
11313
11314
11315         if ((tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG) ||
11316             (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)) {
11317                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
11318                 if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
11319                         tp->write32_rx_mbox = tg3_write_flush_reg32;
11320         }
11321
11322         if (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND) {
11323                 tp->read32 = tg3_read_indirect_reg32;
11324                 tp->write32 = tg3_write_indirect_reg32;
11325                 tp->read32_mbox = tg3_read_indirect_mbox;
11326                 tp->write32_mbox = tg3_write_indirect_mbox;
11327                 tp->write32_tx_mbox = tg3_write_indirect_mbox;
11328                 tp->write32_rx_mbox = tg3_write_indirect_mbox;
11329
11330                 iounmap(tp->regs);
11331                 tp->regs = NULL;
11332
11333                 pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11334                 pci_cmd &= ~PCI_COMMAND_MEMORY;
11335                 pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11336         }
11337         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11338                 tp->read32_mbox = tg3_read32_mbox_5906;
11339                 tp->write32_mbox = tg3_write32_mbox_5906;
11340                 tp->write32_tx_mbox = tg3_write32_mbox_5906;
11341                 tp->write32_rx_mbox = tg3_write32_mbox_5906;
11342         }
11343
11344         if (tp->write32 == tg3_write_indirect_reg32 ||
11345             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
11346              (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11347               GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)))
11348                 tp->tg3_flags |= TG3_FLAG_SRAM_USE_CONFIG;
11349
11350         /* Get eeprom hw config before calling tg3_set_power_state().
11351          * In particular, the TG3_FLG2_IS_NIC flag must be
11352          * determined before calling tg3_set_power_state() so that
11353          * we know whether or not to switch out of Vaux power.
11354          * When the flag is set, it means that GPIO1 is used for eeprom
11355          * write protect and also implies that it is a LOM where GPIOs
11356          * are not used to switch power.
11357          */
11358         tg3_get_eeprom_hw_cfg(tp);
11359
11360         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
11361                 /* Allow reads and writes to the
11362                  * APE register and memory space.
11363                  */
11364                 pci_state_reg |= PCISTATE_ALLOW_APE_CTLSPC_WR |
11365                                  PCISTATE_ALLOW_APE_SHMEM_WR;
11366                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE,
11367                                        pci_state_reg);
11368         }
11369
11370         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11371             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
11372                 tp->tg3_flags |= TG3_FLAG_CPMU_PRESENT;
11373
11374         /* Set up tp->grc_local_ctrl before calling tg3_set_power_state().
11375          * GPIO1 driven high will bring 5700's external PHY out of reset.
11376          * It is also used as eeprom write protect on LOMs.
11377          */
11378         tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
11379         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
11380             (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
11381                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
11382                                        GRC_LCLCTRL_GPIO_OUTPUT1);
11383         /* Unused GPIO3 must be driven as output on 5752 because there
11384          * are no pull-up resistors on unused GPIO pins.
11385          */
11386         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
11387                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
11388
11389         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
11390                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
11391
11392         /* Force the chip into D0. */
11393         err = tg3_set_power_state(tp, PCI_D0);
11394         if (err) {
11395                 printk(KERN_ERR PFX "(%s) transition to D0 failed\n",
11396                        pci_name(tp->pdev));
11397                 return err;
11398         }
11399
11400         /* 5700 B0 chips do not support checksumming correctly due
11401          * to hardware bugs.
11402          */
11403         if (tp->pci_chip_rev_id == CHIPREV_ID_5700_B0)
11404                 tp->tg3_flags |= TG3_FLAG_BROKEN_CHECKSUMS;
11405
11406         /* Derive initial jumbo mode from MTU assigned in
11407          * ether_setup() via the alloc_etherdev() call
11408          */
11409         if (tp->dev->mtu > ETH_DATA_LEN &&
11410             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
11411                 tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
11412
11413         /* Determine WakeOnLan speed to use. */
11414         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11415             tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
11416             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0 ||
11417             tp->pci_chip_rev_id == CHIPREV_ID_5701_B2) {
11418                 tp->tg3_flags &= ~(TG3_FLAG_WOL_SPEED_100MB);
11419         } else {
11420                 tp->tg3_flags |= TG3_FLAG_WOL_SPEED_100MB;
11421         }
11422
11423         /* A few boards don't want Ethernet@WireSpeed phy feature */
11424         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
11425             ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
11426              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) &&
11427              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A1)) ||
11428             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) ||
11429             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
11430                 tp->tg3_flags2 |= TG3_FLG2_NO_ETH_WIRE_SPEED;
11431
11432         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5703_AX ||
11433             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_AX)
11434                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADC_BUG;
11435         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0)
11436                 tp->tg3_flags2 |= TG3_FLG2_PHY_5704_A0_BUG;
11437
11438         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
11439                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11440                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11441                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11442                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
11443                         if (tp->pdev->device != PCI_DEVICE_ID_TIGON3_5756 &&
11444                             tp->pdev->device != PCI_DEVICE_ID_TIGON3_5722)
11445                                 tp->tg3_flags2 |= TG3_FLG2_PHY_JITTER_BUG;
11446                         if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5755M)
11447                                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADJUST_TRIM;
11448                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
11449                         tp->tg3_flags2 |= TG3_FLG2_PHY_BER_BUG;
11450         }
11451
11452         tp->coalesce_mode = 0;
11453         if (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_AX &&
11454             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_BX)
11455                 tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
11456
11457         /* Initialize MAC MI mode, polling disabled. */
11458         tw32_f(MAC_MI_MODE, tp->mi_mode);
11459         udelay(80);
11460
11461         /* Initialize data/descriptor byte/word swapping. */
11462         val = tr32(GRC_MODE);
11463         val &= GRC_MODE_HOST_STACKUP;
11464         tw32(GRC_MODE, val | tp->grc_mode);
11465
11466         tg3_switch_clocks(tp);
11467
11468         /* Clear this out for sanity. */
11469         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
11470
11471         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
11472                               &pci_state_reg);
11473         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
11474             (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) == 0) {
11475                 u32 chiprevid = GET_CHIP_REV_ID(tp->misc_host_ctrl);
11476
11477                 if (chiprevid == CHIPREV_ID_5701_A0 ||
11478                     chiprevid == CHIPREV_ID_5701_B0 ||
11479                     chiprevid == CHIPREV_ID_5701_B2 ||
11480                     chiprevid == CHIPREV_ID_5701_B5) {
11481                         void __iomem *sram_base;
11482
11483                         /* Write some dummy words into the SRAM status block
11484                          * area, see if it reads back correctly.  If the return
11485                          * value is bad, force enable the PCIX workaround.
11486                          */
11487                         sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
11488
11489                         writel(0x00000000, sram_base);
11490                         writel(0x00000000, sram_base + 4);
11491                         writel(0xffffffff, sram_base + 4);
11492                         if (readl(sram_base) != 0x00000000)
11493                                 tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
11494                 }
11495         }
11496
11497         udelay(50);
11498         tg3_nvram_init(tp);
11499
11500         grc_misc_cfg = tr32(GRC_MISC_CFG);
11501         grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
11502
11503         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
11504             (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
11505              grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
11506                 tp->tg3_flags2 |= TG3_FLG2_IS_5788;
11507
11508         if (!(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
11509             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700))
11510                 tp->tg3_flags |= TG3_FLAG_TAGGED_STATUS;
11511         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
11512                 tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
11513                                       HOSTCC_MODE_CLRTICK_TXBD);
11514
11515                 tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
11516                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11517                                        tp->misc_host_ctrl);
11518         }
11519
11520         /* these are limited to 10/100 only */
11521         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
11522              (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
11523             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
11524              tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
11525              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901 ||
11526               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901_2 ||
11527               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5705F)) ||
11528             (tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
11529              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5751F ||
11530               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5753F ||
11531               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5787F)) ||
11532             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11533                 tp->tg3_flags |= TG3_FLAG_10_100_ONLY;
11534
11535         err = tg3_phy_probe(tp);
11536         if (err) {
11537                 printk(KERN_ERR PFX "(%s) phy probe failed, err %d\n",
11538                        pci_name(tp->pdev), err);
11539                 /* ... but do not return immediately ... */
11540         }
11541
11542         tg3_read_partno(tp);
11543         tg3_read_fw_ver(tp);
11544
11545         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
11546                 tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
11547         } else {
11548                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
11549                         tp->tg3_flags |= TG3_FLAG_USE_MI_INTERRUPT;
11550                 else
11551                         tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
11552         }
11553
11554         /* 5700 {AX,BX} chips have a broken status block link
11555          * change bit implementation, so we must use the
11556          * status register in those cases.
11557          */
11558         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
11559                 tp->tg3_flags |= TG3_FLAG_USE_LINKCHG_REG;
11560         else
11561                 tp->tg3_flags &= ~TG3_FLAG_USE_LINKCHG_REG;
11562
11563         /* The led_ctrl is set during tg3_phy_probe, here we might
11564          * have to force the link status polling mechanism based
11565          * upon subsystem IDs.
11566          */
11567         if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
11568             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
11569             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
11570                 tp->tg3_flags |= (TG3_FLAG_USE_MI_INTERRUPT |
11571                                   TG3_FLAG_USE_LINKCHG_REG);
11572         }
11573
11574         /* For all SERDES we poll the MAC status register. */
11575         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
11576                 tp->tg3_flags |= TG3_FLAG_POLL_SERDES;
11577         else
11578                 tp->tg3_flags &= ~TG3_FLAG_POLL_SERDES;
11579
11580         /* All chips before 5787 can get confused if TX buffers
11581          * straddle the 4GB address boundary in some cases.
11582          */
11583         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11584             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11585             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11586             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11587             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11588                 tp->dev->hard_start_xmit = tg3_start_xmit;
11589         else
11590                 tp->dev->hard_start_xmit = tg3_start_xmit_dma_bug;
11591
11592         tp->rx_offset = 2;
11593         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
11594             (tp->tg3_flags & TG3_FLAG_PCIX_MODE) != 0)
11595                 tp->rx_offset = 0;
11596
11597         tp->rx_std_max_post = TG3_RX_RING_SIZE;
11598
11599         /* Increment the rx prod index on the rx std ring by at most
11600          * 8 for these chips to workaround hw errata.
11601          */
11602         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
11603             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
11604             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
11605                 tp->rx_std_max_post = 8;
11606
11607         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND)
11608                 tp->pwrmgmt_thresh = tr32(PCIE_PWR_MGMT_THRESH) &
11609                                      PCIE_PWR_MGMT_L1_THRESH_MSK;
11610
11611         return err;
11612 }
11613
11614 #ifdef CONFIG_SPARC
11615 static int __devinit tg3_get_macaddr_sparc(struct tg3 *tp)
11616 {
11617         struct net_device *dev = tp->dev;
11618         struct pci_dev *pdev = tp->pdev;
11619         struct device_node *dp = pci_device_to_OF_node(pdev);
11620         const unsigned char *addr;
11621         int len;
11622
11623         addr = of_get_property(dp, "local-mac-address", &len);
11624         if (addr && len == 6) {
11625                 memcpy(dev->dev_addr, addr, 6);
11626                 memcpy(dev->perm_addr, dev->dev_addr, 6);
11627                 return 0;
11628         }
11629         return -ENODEV;
11630 }
11631
11632 static int __devinit tg3_get_default_macaddr_sparc(struct tg3 *tp)
11633 {
11634         struct net_device *dev = tp->dev;
11635
11636         memcpy(dev->dev_addr, idprom->id_ethaddr, 6);
11637         memcpy(dev->perm_addr, idprom->id_ethaddr, 6);
11638         return 0;
11639 }
11640 #endif
11641
11642 static int __devinit tg3_get_device_address(struct tg3 *tp)
11643 {
11644         struct net_device *dev = tp->dev;
11645         u32 hi, lo, mac_offset;
11646         int addr_ok = 0;
11647
11648 #ifdef CONFIG_SPARC
11649         if (!tg3_get_macaddr_sparc(tp))
11650                 return 0;
11651 #endif
11652
11653         mac_offset = 0x7c;
11654         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
11655             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
11656                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
11657                         mac_offset = 0xcc;
11658                 if (tg3_nvram_lock(tp))
11659                         tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
11660                 else
11661                         tg3_nvram_unlock(tp);
11662         }
11663         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11664                 mac_offset = 0x10;
11665
11666         /* First try to get it from MAC address mailbox. */
11667         tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
11668         if ((hi >> 16) == 0x484b) {
11669                 dev->dev_addr[0] = (hi >>  8) & 0xff;
11670                 dev->dev_addr[1] = (hi >>  0) & 0xff;
11671
11672                 tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
11673                 dev->dev_addr[2] = (lo >> 24) & 0xff;
11674                 dev->dev_addr[3] = (lo >> 16) & 0xff;
11675                 dev->dev_addr[4] = (lo >>  8) & 0xff;
11676                 dev->dev_addr[5] = (lo >>  0) & 0xff;
11677
11678                 /* Some old bootcode may report a 0 MAC address in SRAM */
11679                 addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
11680         }
11681         if (!addr_ok) {
11682                 /* Next, try NVRAM. */
11683                 if (!tg3_nvram_read(tp, mac_offset + 0, &hi) &&
11684                     !tg3_nvram_read(tp, mac_offset + 4, &lo)) {
11685                         dev->dev_addr[0] = ((hi >> 16) & 0xff);
11686                         dev->dev_addr[1] = ((hi >> 24) & 0xff);
11687                         dev->dev_addr[2] = ((lo >>  0) & 0xff);
11688                         dev->dev_addr[3] = ((lo >>  8) & 0xff);
11689                         dev->dev_addr[4] = ((lo >> 16) & 0xff);
11690                         dev->dev_addr[5] = ((lo >> 24) & 0xff);
11691                 }
11692                 /* Finally just fetch it out of the MAC control regs. */
11693                 else {
11694                         hi = tr32(MAC_ADDR_0_HIGH);
11695                         lo = tr32(MAC_ADDR_0_LOW);
11696
11697                         dev->dev_addr[5] = lo & 0xff;
11698                         dev->dev_addr[4] = (lo >> 8) & 0xff;
11699                         dev->dev_addr[3] = (lo >> 16) & 0xff;
11700                         dev->dev_addr[2] = (lo >> 24) & 0xff;
11701                         dev->dev_addr[1] = hi & 0xff;
11702                         dev->dev_addr[0] = (hi >> 8) & 0xff;
11703                 }
11704         }
11705
11706         if (!is_valid_ether_addr(&dev->dev_addr[0])) {
11707 #ifdef CONFIG_SPARC64
11708                 if (!tg3_get_default_macaddr_sparc(tp))
11709                         return 0;
11710 #endif
11711                 return -EINVAL;
11712         }
11713         memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
11714         return 0;
11715 }
11716
11717 #define BOUNDARY_SINGLE_CACHELINE       1
11718 #define BOUNDARY_MULTI_CACHELINE        2
11719
11720 static u32 __devinit tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
11721 {
11722         int cacheline_size;
11723         u8 byte;
11724         int goal;
11725
11726         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
11727         if (byte == 0)
11728                 cacheline_size = 1024;
11729         else
11730                 cacheline_size = (int) byte * 4;
11731
11732         /* On 5703 and later chips, the boundary bits have no
11733          * effect.
11734          */
11735         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
11736             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701 &&
11737             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
11738                 goto out;
11739
11740 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
11741         goal = BOUNDARY_MULTI_CACHELINE;
11742 #else
11743 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
11744         goal = BOUNDARY_SINGLE_CACHELINE;
11745 #else
11746         goal = 0;
11747 #endif
11748 #endif
11749
11750         if (!goal)
11751                 goto out;
11752
11753         /* PCI controllers on most RISC systems tend to disconnect
11754          * when a device tries to burst across a cache-line boundary.
11755          * Therefore, letting tg3 do so just wastes PCI bandwidth.
11756          *
11757          * Unfortunately, for PCI-E there are only limited
11758          * write-side controls for this, and thus for reads
11759          * we will still get the disconnects.  We'll also waste
11760          * these PCI cycles for both read and write for chips
11761          * other than 5700 and 5701 which do not implement the
11762          * boundary bits.
11763          */
11764         if ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
11765             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
11766                 switch (cacheline_size) {
11767                 case 16:
11768                 case 32:
11769                 case 64:
11770                 case 128:
11771                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11772                                 val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
11773                                         DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
11774                         } else {
11775                                 val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
11776                                         DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
11777                         }
11778                         break;
11779
11780                 case 256:
11781                         val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
11782                                 DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
11783                         break;
11784
11785                 default:
11786                         val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
11787                                 DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
11788                         break;
11789                 };
11790         } else if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
11791                 switch (cacheline_size) {
11792                 case 16:
11793                 case 32:
11794                 case 64:
11795                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11796                                 val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
11797                                 val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
11798                                 break;
11799                         }
11800                         /* fallthrough */
11801                 case 128:
11802                 default:
11803                         val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
11804                         val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
11805                         break;
11806                 };
11807         } else {
11808                 switch (cacheline_size) {
11809                 case 16:
11810                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11811                                 val |= (DMA_RWCTRL_READ_BNDRY_16 |
11812                                         DMA_RWCTRL_WRITE_BNDRY_16);
11813                                 break;
11814                         }
11815                         /* fallthrough */
11816                 case 32:
11817                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11818                                 val |= (DMA_RWCTRL_READ_BNDRY_32 |
11819                                         DMA_RWCTRL_WRITE_BNDRY_32);
11820                                 break;
11821                         }
11822                         /* fallthrough */
11823                 case 64:
11824                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11825                                 val |= (DMA_RWCTRL_READ_BNDRY_64 |
11826                                         DMA_RWCTRL_WRITE_BNDRY_64);
11827                                 break;
11828                         }
11829                         /* fallthrough */
11830                 case 128:
11831                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11832                                 val |= (DMA_RWCTRL_READ_BNDRY_128 |
11833                                         DMA_RWCTRL_WRITE_BNDRY_128);
11834                                 break;
11835                         }
11836                         /* fallthrough */
11837                 case 256:
11838                         val |= (DMA_RWCTRL_READ_BNDRY_256 |
11839                                 DMA_RWCTRL_WRITE_BNDRY_256);
11840                         break;
11841                 case 512:
11842                         val |= (DMA_RWCTRL_READ_BNDRY_512 |
11843                                 DMA_RWCTRL_WRITE_BNDRY_512);
11844                         break;
11845                 case 1024:
11846                 default:
11847                         val |= (DMA_RWCTRL_READ_BNDRY_1024 |
11848                                 DMA_RWCTRL_WRITE_BNDRY_1024);
11849                         break;
11850                 };
11851         }
11852
11853 out:
11854         return val;
11855 }
11856
11857 static int __devinit tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma, int size, int to_device)
11858 {
11859         struct tg3_internal_buffer_desc test_desc;
11860         u32 sram_dma_descs;
11861         int i, ret;
11862
11863         sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
11864
11865         tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
11866         tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
11867         tw32(RDMAC_STATUS, 0);
11868         tw32(WDMAC_STATUS, 0);
11869
11870         tw32(BUFMGR_MODE, 0);
11871         tw32(FTQ_RESET, 0);
11872
11873         test_desc.addr_hi = ((u64) buf_dma) >> 32;
11874         test_desc.addr_lo = buf_dma & 0xffffffff;
11875         test_desc.nic_mbuf = 0x00002100;
11876         test_desc.len = size;
11877
11878         /*
11879          * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
11880          * the *second* time the tg3 driver was getting loaded after an
11881          * initial scan.
11882          *
11883          * Broadcom tells me:
11884          *   ...the DMA engine is connected to the GRC block and a DMA
11885          *   reset may affect the GRC block in some unpredictable way...
11886          *   The behavior of resets to individual blocks has not been tested.
11887          *
11888          * Broadcom noted the GRC reset will also reset all sub-components.
11889          */
11890         if (to_device) {
11891                 test_desc.cqid_sqid = (13 << 8) | 2;
11892
11893                 tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
11894                 udelay(40);
11895         } else {
11896                 test_desc.cqid_sqid = (16 << 8) | 7;
11897
11898                 tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
11899                 udelay(40);
11900         }
11901         test_desc.flags = 0x00000005;
11902
11903         for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
11904                 u32 val;
11905
11906                 val = *(((u32 *)&test_desc) + i);
11907                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
11908                                        sram_dma_descs + (i * sizeof(u32)));
11909                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
11910         }
11911         pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
11912
11913         if (to_device) {
11914                 tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
11915         } else {
11916                 tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
11917         }
11918
11919         ret = -ENODEV;
11920         for (i = 0; i < 40; i++) {
11921                 u32 val;
11922
11923                 if (to_device)
11924                         val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
11925                 else
11926                         val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
11927                 if ((val & 0xffff) == sram_dma_descs) {
11928                         ret = 0;
11929                         break;
11930                 }
11931
11932                 udelay(100);
11933         }
11934
11935         return ret;
11936 }
11937
11938 #define TEST_BUFFER_SIZE        0x2000
11939
11940 static int __devinit tg3_test_dma(struct tg3 *tp)
11941 {
11942         dma_addr_t buf_dma;
11943         u32 *buf, saved_dma_rwctrl;
11944         int ret;
11945
11946         buf = pci_alloc_consistent(tp->pdev, TEST_BUFFER_SIZE, &buf_dma);
11947         if (!buf) {
11948                 ret = -ENOMEM;
11949                 goto out_nofree;
11950         }
11951
11952         tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
11953                           (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
11954
11955         tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
11956
11957         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
11958                 /* DMA read watermark not used on PCIE */
11959                 tp->dma_rwctrl |= 0x00180000;
11960         } else if (!(tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
11961                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 ||
11962                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)
11963                         tp->dma_rwctrl |= 0x003f0000;
11964                 else
11965                         tp->dma_rwctrl |= 0x003f000f;
11966         } else {
11967                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
11968                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
11969                         u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
11970                         u32 read_water = 0x7;
11971
11972                         /* If the 5704 is behind the EPB bridge, we can
11973                          * do the less restrictive ONE_DMA workaround for
11974                          * better performance.
11975                          */
11976                         if ((tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) &&
11977                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
11978                                 tp->dma_rwctrl |= 0x8000;
11979                         else if (ccval == 0x6 || ccval == 0x7)
11980                                 tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
11981
11982                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703)
11983                                 read_water = 4;
11984                         /* Set bit 23 to enable PCIX hw bug fix */
11985                         tp->dma_rwctrl |=
11986                                 (read_water << DMA_RWCTRL_READ_WATER_SHIFT) |
11987                                 (0x3 << DMA_RWCTRL_WRITE_WATER_SHIFT) |
11988                                 (1 << 23);
11989                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780) {
11990                         /* 5780 always in PCIX mode */
11991                         tp->dma_rwctrl |= 0x00144000;
11992                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
11993                         /* 5714 always in PCIX mode */
11994                         tp->dma_rwctrl |= 0x00148000;
11995                 } else {
11996                         tp->dma_rwctrl |= 0x001b000f;
11997                 }
11998         }
11999
12000         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
12001             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
12002                 tp->dma_rwctrl &= 0xfffffff0;
12003
12004         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12005             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
12006                 /* Remove this if it causes problems for some boards. */
12007                 tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
12008
12009                 /* On 5700/5701 chips, we need to set this bit.
12010                  * Otherwise the chip will issue cacheline transactions
12011                  * to streamable DMA memory with not all the byte
12012                  * enables turned on.  This is an error on several
12013                  * RISC PCI controllers, in particular sparc64.
12014                  *
12015                  * On 5703/5704 chips, this bit has been reassigned
12016                  * a different meaning.  In particular, it is used
12017                  * on those chips to enable a PCI-X workaround.
12018                  */
12019                 tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
12020         }
12021
12022         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12023
12024 #if 0
12025         /* Unneeded, already done by tg3_get_invariants.  */
12026         tg3_switch_clocks(tp);
12027 #endif
12028
12029         ret = 0;
12030         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
12031             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
12032                 goto out;
12033
12034         /* It is best to perform DMA test with maximum write burst size
12035          * to expose the 5700/5701 write DMA bug.
12036          */
12037         saved_dma_rwctrl = tp->dma_rwctrl;
12038         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12039         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12040
12041         while (1) {
12042                 u32 *p = buf, i;
12043
12044                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
12045                         p[i] = i;
12046
12047                 /* Send the buffer to the chip. */
12048                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 1);
12049                 if (ret) {
12050                         printk(KERN_ERR "tg3_test_dma() Write the buffer failed %d\n", ret);
12051                         break;
12052                 }
12053
12054 #if 0
12055                 /* validate data reached card RAM correctly. */
12056                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
12057                         u32 val;
12058                         tg3_read_mem(tp, 0x2100 + (i*4), &val);
12059                         if (le32_to_cpu(val) != p[i]) {
12060                                 printk(KERN_ERR "  tg3_test_dma()  Card buffer corrupted on write! (%d != %d)\n", val, i);
12061                                 /* ret = -ENODEV here? */
12062                         }
12063                         p[i] = 0;
12064                 }
12065 #endif
12066                 /* Now read it back. */
12067                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 0);
12068                 if (ret) {
12069                         printk(KERN_ERR "tg3_test_dma() Read the buffer failed %d\n", ret);
12070
12071                         break;
12072                 }
12073
12074                 /* Verify it. */
12075                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
12076                         if (p[i] == i)
12077                                 continue;
12078
12079                         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
12080                             DMA_RWCTRL_WRITE_BNDRY_16) {
12081                                 tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12082                                 tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
12083                                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12084                                 break;
12085                         } else {
12086                                 printk(KERN_ERR "tg3_test_dma() buffer corrupted on read back! (%d != %d)\n", p[i], i);
12087                                 ret = -ENODEV;
12088                                 goto out;
12089                         }
12090                 }
12091
12092                 if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
12093                         /* Success. */
12094                         ret = 0;
12095                         break;
12096                 }
12097         }
12098         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
12099             DMA_RWCTRL_WRITE_BNDRY_16) {
12100                 static struct pci_device_id dma_wait_state_chipsets[] = {
12101                         { PCI_DEVICE(PCI_VENDOR_ID_APPLE,
12102                                      PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
12103                         { },
12104                 };
12105
12106                 /* DMA test passed without adjusting DMA boundary,
12107                  * now look for chipsets that are known to expose the
12108                  * DMA bug without failing the test.
12109                  */
12110                 if (pci_dev_present(dma_wait_state_chipsets)) {
12111                         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12112                         tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
12113                 }
12114                 else
12115                         /* Safe to use the calculated DMA boundary. */
12116                         tp->dma_rwctrl = saved_dma_rwctrl;
12117
12118                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12119         }
12120
12121 out:
12122         pci_free_consistent(tp->pdev, TEST_BUFFER_SIZE, buf, buf_dma);
12123 out_nofree:
12124         return ret;
12125 }
12126
12127 static void __devinit tg3_init_link_config(struct tg3 *tp)
12128 {
12129         tp->link_config.advertising =
12130                 (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
12131                  ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
12132                  ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
12133                  ADVERTISED_Autoneg | ADVERTISED_MII);
12134         tp->link_config.speed = SPEED_INVALID;
12135         tp->link_config.duplex = DUPLEX_INVALID;
12136         tp->link_config.autoneg = AUTONEG_ENABLE;
12137         tp->link_config.active_speed = SPEED_INVALID;
12138         tp->link_config.active_duplex = DUPLEX_INVALID;
12139         tp->link_config.phy_is_low_power = 0;
12140         tp->link_config.orig_speed = SPEED_INVALID;
12141         tp->link_config.orig_duplex = DUPLEX_INVALID;
12142         tp->link_config.orig_autoneg = AUTONEG_INVALID;
12143 }
12144
12145 static void __devinit tg3_init_bufmgr_config(struct tg3 *tp)
12146 {
12147         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
12148                 tp->bufmgr_config.mbuf_read_dma_low_water =
12149                         DEFAULT_MB_RDMA_LOW_WATER_5705;
12150                 tp->bufmgr_config.mbuf_mac_rx_low_water =
12151                         DEFAULT_MB_MACRX_LOW_WATER_5705;
12152                 tp->bufmgr_config.mbuf_high_water =
12153                         DEFAULT_MB_HIGH_WATER_5705;
12154                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
12155                         tp->bufmgr_config.mbuf_mac_rx_low_water =
12156                                 DEFAULT_MB_MACRX_LOW_WATER_5906;
12157                         tp->bufmgr_config.mbuf_high_water =
12158                                 DEFAULT_MB_HIGH_WATER_5906;
12159                 }
12160
12161                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
12162                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
12163                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
12164                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
12165                 tp->bufmgr_config.mbuf_high_water_jumbo =
12166                         DEFAULT_MB_HIGH_WATER_JUMBO_5780;
12167         } else {
12168                 tp->bufmgr_config.mbuf_read_dma_low_water =
12169                         DEFAULT_MB_RDMA_LOW_WATER;
12170                 tp->bufmgr_config.mbuf_mac_rx_low_water =
12171                         DEFAULT_MB_MACRX_LOW_WATER;
12172                 tp->bufmgr_config.mbuf_high_water =
12173                         DEFAULT_MB_HIGH_WATER;
12174
12175                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
12176                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
12177                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
12178                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
12179                 tp->bufmgr_config.mbuf_high_water_jumbo =
12180                         DEFAULT_MB_HIGH_WATER_JUMBO;
12181         }
12182
12183         tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
12184         tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
12185 }
12186
12187 static char * __devinit tg3_phy_string(struct tg3 *tp)
12188 {
12189         switch (tp->phy_id & PHY_ID_MASK) {
12190         case PHY_ID_BCM5400:    return "5400";
12191         case PHY_ID_BCM5401:    return "5401";
12192         case PHY_ID_BCM5411:    return "5411";
12193         case PHY_ID_BCM5701:    return "5701";
12194         case PHY_ID_BCM5703:    return "5703";
12195         case PHY_ID_BCM5704:    return "5704";
12196         case PHY_ID_BCM5705:    return "5705";
12197         case PHY_ID_BCM5750:    return "5750";
12198         case PHY_ID_BCM5752:    return "5752";
12199         case PHY_ID_BCM5714:    return "5714";
12200         case PHY_ID_BCM5780:    return "5780";
12201         case PHY_ID_BCM5755:    return "5755";
12202         case PHY_ID_BCM5787:    return "5787";
12203         case PHY_ID_BCM5784:    return "5784";
12204         case PHY_ID_BCM5756:    return "5722/5756";
12205         case PHY_ID_BCM5906:    return "5906";
12206         case PHY_ID_BCM5761:    return "5761";
12207         case PHY_ID_BCM8002:    return "8002/serdes";
12208         case 0:                 return "serdes";
12209         default:                return "unknown";
12210         };
12211 }
12212
12213 static char * __devinit tg3_bus_string(struct tg3 *tp, char *str)
12214 {
12215         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
12216                 strcpy(str, "PCI Express");
12217                 return str;
12218         } else if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
12219                 u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
12220
12221                 strcpy(str, "PCIX:");
12222
12223                 if ((clock_ctrl == 7) ||
12224                     ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
12225                      GRC_MISC_CFG_BOARD_ID_5704CIOBE))
12226                         strcat(str, "133MHz");
12227                 else if (clock_ctrl == 0)
12228                         strcat(str, "33MHz");
12229                 else if (clock_ctrl == 2)
12230                         strcat(str, "50MHz");
12231                 else if (clock_ctrl == 4)
12232                         strcat(str, "66MHz");
12233                 else if (clock_ctrl == 6)
12234                         strcat(str, "100MHz");
12235         } else {
12236                 strcpy(str, "PCI:");
12237                 if (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED)
12238                         strcat(str, "66MHz");
12239                 else
12240                         strcat(str, "33MHz");
12241         }
12242         if (tp->tg3_flags & TG3_FLAG_PCI_32BIT)
12243                 strcat(str, ":32-bit");
12244         else
12245                 strcat(str, ":64-bit");
12246         return str;
12247 }
12248
12249 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *tp)
12250 {
12251         struct pci_dev *peer;
12252         unsigned int func, devnr = tp->pdev->devfn & ~7;
12253
12254         for (func = 0; func < 8; func++) {
12255                 peer = pci_get_slot(tp->pdev->bus, devnr | func);
12256                 if (peer && peer != tp->pdev)
12257                         break;
12258                 pci_dev_put(peer);
12259         }
12260         /* 5704 can be configured in single-port mode, set peer to
12261          * tp->pdev in that case.
12262          */
12263         if (!peer) {
12264                 peer = tp->pdev;
12265                 return peer;
12266         }
12267
12268         /*
12269          * We don't need to keep the refcount elevated; there's no way
12270          * to remove one half of this device without removing the other
12271          */
12272         pci_dev_put(peer);
12273
12274         return peer;
12275 }
12276
12277 static void __devinit tg3_init_coal(struct tg3 *tp)
12278 {
12279         struct ethtool_coalesce *ec = &tp->coal;
12280
12281         memset(ec, 0, sizeof(*ec));
12282         ec->cmd = ETHTOOL_GCOALESCE;
12283         ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
12284         ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
12285         ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
12286         ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
12287         ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
12288         ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
12289         ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
12290         ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
12291         ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
12292
12293         if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
12294                                  HOSTCC_MODE_CLRTICK_TXBD)) {
12295                 ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
12296                 ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
12297                 ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
12298                 ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
12299         }
12300
12301         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
12302                 ec->rx_coalesce_usecs_irq = 0;
12303                 ec->tx_coalesce_usecs_irq = 0;
12304                 ec->stats_block_coalesce_usecs = 0;
12305         }
12306 }
12307
12308 static int __devinit tg3_init_one(struct pci_dev *pdev,
12309                                   const struct pci_device_id *ent)
12310 {
12311         static int tg3_version_printed = 0;
12312         unsigned long tg3reg_base, tg3reg_len;
12313         struct net_device *dev;
12314         struct tg3 *tp;
12315         int i, err, pm_cap;
12316         char str[40];
12317         u64 dma_mask, persist_dma_mask;
12318
12319         if (tg3_version_printed++ == 0)
12320                 printk(KERN_INFO "%s", version);
12321
12322         err = pci_enable_device(pdev);
12323         if (err) {
12324                 printk(KERN_ERR PFX "Cannot enable PCI device, "
12325                        "aborting.\n");
12326                 return err;
12327         }
12328
12329         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12330                 printk(KERN_ERR PFX "Cannot find proper PCI device "
12331                        "base address, aborting.\n");
12332                 err = -ENODEV;
12333                 goto err_out_disable_pdev;
12334         }
12335
12336         err = pci_request_regions(pdev, DRV_MODULE_NAME);
12337         if (err) {
12338                 printk(KERN_ERR PFX "Cannot obtain PCI resources, "
12339                        "aborting.\n");
12340                 goto err_out_disable_pdev;
12341         }
12342
12343         pci_set_master(pdev);
12344
12345         /* Find power-management capability. */
12346         pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
12347         if (pm_cap == 0) {
12348                 printk(KERN_ERR PFX "Cannot find PowerManagement capability, "
12349                        "aborting.\n");
12350                 err = -EIO;
12351                 goto err_out_free_res;
12352         }
12353
12354         tg3reg_base = pci_resource_start(pdev, 0);
12355         tg3reg_len = pci_resource_len(pdev, 0);
12356
12357         dev = alloc_etherdev(sizeof(*tp));
12358         if (!dev) {
12359                 printk(KERN_ERR PFX "Etherdev alloc failed, aborting.\n");
12360                 err = -ENOMEM;
12361                 goto err_out_free_res;
12362         }
12363
12364         SET_NETDEV_DEV(dev, &pdev->dev);
12365
12366 #if TG3_VLAN_TAG_USED
12367         dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
12368         dev->vlan_rx_register = tg3_vlan_rx_register;
12369 #endif
12370
12371         tp = netdev_priv(dev);
12372         tp->pdev = pdev;
12373         tp->dev = dev;
12374         tp->pm_cap = pm_cap;
12375         tp->mac_mode = TG3_DEF_MAC_MODE;
12376         tp->rx_mode = TG3_DEF_RX_MODE;
12377         tp->tx_mode = TG3_DEF_TX_MODE;
12378         tp->mi_mode = MAC_MI_MODE_BASE;
12379         if (tg3_debug > 0)
12380                 tp->msg_enable = tg3_debug;
12381         else
12382                 tp->msg_enable = TG3_DEF_MSG_ENABLE;
12383
12384         /* The word/byte swap controls here control register access byte
12385          * swapping.  DMA data byte swapping is controlled in the GRC_MODE
12386          * setting below.
12387          */
12388         tp->misc_host_ctrl =
12389                 MISC_HOST_CTRL_MASK_PCI_INT |
12390                 MISC_HOST_CTRL_WORD_SWAP |
12391                 MISC_HOST_CTRL_INDIR_ACCESS |
12392                 MISC_HOST_CTRL_PCISTATE_RW;
12393
12394         /* The NONFRM (non-frame) byte/word swap controls take effect
12395          * on descriptor entries, anything which isn't packet data.
12396          *
12397          * The StrongARM chips on the board (one for tx, one for rx)
12398          * are running in big-endian mode.
12399          */
12400         tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
12401                         GRC_MODE_WSWAP_NONFRM_DATA);
12402 #ifdef __BIG_ENDIAN
12403         tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
12404 #endif
12405         spin_lock_init(&tp->lock);
12406         spin_lock_init(&tp->indirect_lock);
12407         INIT_WORK(&tp->reset_task, tg3_reset_task);
12408
12409         tp->regs = ioremap_nocache(tg3reg_base, tg3reg_len);
12410         if (!tp->regs) {
12411                 printk(KERN_ERR PFX "Cannot map device registers, "
12412                        "aborting.\n");
12413                 err = -ENOMEM;
12414                 goto err_out_free_dev;
12415         }
12416
12417         tg3_init_link_config(tp);
12418
12419         tp->rx_pending = TG3_DEF_RX_RING_PENDING;
12420         tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
12421         tp->tx_pending = TG3_DEF_TX_RING_PENDING;
12422
12423         dev->open = tg3_open;
12424         dev->stop = tg3_close;
12425         dev->get_stats = tg3_get_stats;
12426         dev->set_multicast_list = tg3_set_rx_mode;
12427         dev->set_mac_address = tg3_set_mac_addr;
12428         dev->do_ioctl = tg3_ioctl;
12429         dev->tx_timeout = tg3_tx_timeout;
12430         netif_napi_add(dev, &tp->napi, tg3_poll, 64);
12431         dev->ethtool_ops = &tg3_ethtool_ops;
12432         dev->watchdog_timeo = TG3_TX_TIMEOUT;
12433         dev->change_mtu = tg3_change_mtu;
12434         dev->irq = pdev->irq;
12435 #ifdef CONFIG_NET_POLL_CONTROLLER
12436         dev->poll_controller = tg3_poll_controller;
12437 #endif
12438
12439         err = tg3_get_invariants(tp);
12440         if (err) {
12441                 printk(KERN_ERR PFX "Problem fetching invariants of chip, "
12442                        "aborting.\n");
12443                 goto err_out_iounmap;
12444         }
12445
12446         /* The EPB bridge inside 5714, 5715, and 5780 and any
12447          * device behind the EPB cannot support DMA addresses > 40-bit.
12448          * On 64-bit systems with IOMMU, use 40-bit dma_mask.
12449          * On 64-bit systems without IOMMU, use 64-bit dma_mask and
12450          * do DMA address check in tg3_start_xmit().
12451          */
12452         if (tp->tg3_flags2 & TG3_FLG2_IS_5788)
12453                 persist_dma_mask = dma_mask = DMA_32BIT_MASK;
12454         else if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) {
12455                 persist_dma_mask = dma_mask = DMA_40BIT_MASK;
12456 #ifdef CONFIG_HIGHMEM
12457                 dma_mask = DMA_64BIT_MASK;
12458 #endif
12459         } else
12460                 persist_dma_mask = dma_mask = DMA_64BIT_MASK;
12461
12462         /* Configure DMA attributes. */
12463         if (dma_mask > DMA_32BIT_MASK) {
12464                 err = pci_set_dma_mask(pdev, dma_mask);
12465                 if (!err) {
12466                         dev->features |= NETIF_F_HIGHDMA;
12467                         err = pci_set_consistent_dma_mask(pdev,
12468                                                           persist_dma_mask);
12469                         if (err < 0) {
12470                                 printk(KERN_ERR PFX "Unable to obtain 64 bit "
12471                                        "DMA for consistent allocations\n");
12472                                 goto err_out_iounmap;
12473                         }
12474                 }
12475         }
12476         if (err || dma_mask == DMA_32BIT_MASK) {
12477                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
12478                 if (err) {
12479                         printk(KERN_ERR PFX "No usable DMA configuration, "
12480                                "aborting.\n");
12481                         goto err_out_iounmap;
12482                 }
12483         }
12484
12485         tg3_init_bufmgr_config(tp);
12486
12487         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
12488                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
12489         }
12490         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12491             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
12492             tp->pci_chip_rev_id == CHIPREV_ID_5705_A0 ||
12493             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
12494             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
12495                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
12496         } else {
12497                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE | TG3_FLG2_TSO_BUG;
12498         }
12499
12500         /* TSO is on by default on chips that support hardware TSO.
12501          * Firmware TSO on older chips gives lower performance, so it
12502          * is off by default, but can be enabled using ethtool.
12503          */
12504         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
12505                 dev->features |= NETIF_F_TSO;
12506                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO_2) &&
12507                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906))
12508                         dev->features |= NETIF_F_TSO6;
12509                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
12510                         dev->features |= NETIF_F_TSO_ECN;
12511         }
12512
12513
12514         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 &&
12515             !(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) &&
12516             !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
12517                 tp->tg3_flags2 |= TG3_FLG2_MAX_RXPEND_64;
12518                 tp->rx_pending = 63;
12519         }
12520
12521         err = tg3_get_device_address(tp);
12522         if (err) {
12523                 printk(KERN_ERR PFX "Could not obtain valid ethernet address, "
12524                        "aborting.\n");
12525                 goto err_out_iounmap;
12526         }
12527
12528         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
12529                 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
12530                         printk(KERN_ERR PFX "Cannot find proper PCI device "
12531                                "base address for APE, aborting.\n");
12532                         err = -ENODEV;
12533                         goto err_out_iounmap;
12534                 }
12535
12536                 tg3reg_base = pci_resource_start(pdev, 2);
12537                 tg3reg_len = pci_resource_len(pdev, 2);
12538
12539                 tp->aperegs = ioremap_nocache(tg3reg_base, tg3reg_len);
12540                 if (tp->aperegs == 0UL) {
12541                         printk(KERN_ERR PFX "Cannot map APE registers, "
12542                                "aborting.\n");
12543                         err = -ENOMEM;
12544                         goto err_out_iounmap;
12545                 }
12546
12547                 tg3_ape_lock_init(tp);
12548         }
12549
12550         /*
12551          * Reset chip in case UNDI or EFI driver did not shutdown
12552          * DMA self test will enable WDMAC and we'll see (spurious)
12553          * pending DMA on the PCI bus at that point.
12554          */
12555         if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
12556             (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
12557                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
12558                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12559         }
12560
12561         err = tg3_test_dma(tp);
12562         if (err) {
12563                 printk(KERN_ERR PFX "DMA engine test failed, aborting.\n");
12564                 goto err_out_apeunmap;
12565         }
12566
12567         /* Tigon3 can do ipv4 only... and some chips have buggy
12568          * checksumming.
12569          */
12570         if ((tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) == 0) {
12571                 dev->features |= NETIF_F_IP_CSUM | NETIF_F_SG;
12572                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
12573                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
12574                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
12575                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
12576                         dev->features |= NETIF_F_IPV6_CSUM;
12577
12578                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
12579         } else
12580                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
12581
12582         /* flow control autonegotiation is default behavior */
12583         tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
12584
12585         tg3_init_coal(tp);
12586
12587         pci_set_drvdata(pdev, dev);
12588
12589         err = register_netdev(dev);
12590         if (err) {
12591                 printk(KERN_ERR PFX "Cannot register net device, "
12592                        "aborting.\n");
12593                 goto err_out_apeunmap;
12594         }
12595
12596         printk(KERN_INFO "%s: Tigon3 [partno(%s) rev %04x PHY(%s)] (%s) %s Ethernet ",
12597                dev->name,
12598                tp->board_part_number,
12599                tp->pci_chip_rev_id,
12600                tg3_phy_string(tp),
12601                tg3_bus_string(tp, str),
12602                ((tp->tg3_flags & TG3_FLAG_10_100_ONLY) ? "10/100Base-TX" :
12603                 ((tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) ? "1000Base-SX" :
12604                  "10/100/1000Base-T")));
12605
12606         for (i = 0; i < 6; i++)
12607                 printk("%2.2x%c", dev->dev_addr[i],
12608                        i == 5 ? '\n' : ':');
12609
12610         printk(KERN_INFO "%s: RXcsums[%d] LinkChgREG[%d] "
12611                "MIirq[%d] ASF[%d] WireSpeed[%d] TSOcap[%d]\n",
12612                dev->name,
12613                (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0,
12614                (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) != 0,
12615                (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) != 0,
12616                (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0,
12617                (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED) == 0,
12618                (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) != 0);
12619         printk(KERN_INFO "%s: dma_rwctrl[%08x] dma_mask[%d-bit]\n",
12620                dev->name, tp->dma_rwctrl,
12621                (pdev->dma_mask == DMA_32BIT_MASK) ? 32 :
12622                 (((u64) pdev->dma_mask == DMA_40BIT_MASK) ? 40 : 64));
12623
12624         return 0;
12625
12626 err_out_apeunmap:
12627         if (tp->aperegs) {
12628                 iounmap(tp->aperegs);
12629                 tp->aperegs = NULL;
12630         }
12631
12632 err_out_iounmap:
12633         if (tp->regs) {
12634                 iounmap(tp->regs);
12635                 tp->regs = NULL;
12636         }
12637
12638 err_out_free_dev:
12639         free_netdev(dev);
12640
12641 err_out_free_res:
12642         pci_release_regions(pdev);
12643
12644 err_out_disable_pdev:
12645         pci_disable_device(pdev);
12646         pci_set_drvdata(pdev, NULL);
12647         return err;
12648 }
12649
12650 static void __devexit tg3_remove_one(struct pci_dev *pdev)
12651 {
12652         struct net_device *dev = pci_get_drvdata(pdev);
12653
12654         if (dev) {
12655                 struct tg3 *tp = netdev_priv(dev);
12656
12657                 flush_scheduled_work();
12658                 unregister_netdev(dev);
12659                 if (tp->aperegs) {
12660                         iounmap(tp->aperegs);
12661                         tp->aperegs = NULL;
12662                 }
12663                 if (tp->regs) {
12664                         iounmap(tp->regs);
12665                         tp->regs = NULL;
12666                 }
12667                 free_netdev(dev);
12668                 pci_release_regions(pdev);
12669                 pci_disable_device(pdev);
12670                 pci_set_drvdata(pdev, NULL);
12671         }
12672 }
12673
12674 static int tg3_suspend(struct pci_dev *pdev, pm_message_t state)
12675 {
12676         struct net_device *dev = pci_get_drvdata(pdev);
12677         struct tg3 *tp = netdev_priv(dev);
12678         int err;
12679
12680         /* PCI register 4 needs to be saved whether netif_running() or not.
12681          * MSI address and data need to be saved if using MSI and
12682          * netif_running().
12683          */
12684         pci_save_state(pdev);
12685
12686         if (!netif_running(dev))
12687                 return 0;
12688
12689         flush_scheduled_work();
12690         tg3_netif_stop(tp);
12691
12692         del_timer_sync(&tp->timer);
12693
12694         tg3_full_lock(tp, 1);
12695         tg3_disable_ints(tp);
12696         tg3_full_unlock(tp);
12697
12698         netif_device_detach(dev);
12699
12700         tg3_full_lock(tp, 0);
12701         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12702         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
12703         tg3_full_unlock(tp);
12704
12705         err = tg3_set_power_state(tp, pci_choose_state(pdev, state));
12706         if (err) {
12707                 tg3_full_lock(tp, 0);
12708
12709                 tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
12710                 if (tg3_restart_hw(tp, 1))
12711                         goto out;
12712
12713                 tp->timer.expires = jiffies + tp->timer_offset;
12714                 add_timer(&tp->timer);
12715
12716                 netif_device_attach(dev);
12717                 tg3_netif_start(tp);
12718
12719 out:
12720                 tg3_full_unlock(tp);
12721         }
12722
12723         return err;
12724 }
12725
12726 static int tg3_resume(struct pci_dev *pdev)
12727 {
12728         struct net_device *dev = pci_get_drvdata(pdev);
12729         struct tg3 *tp = netdev_priv(dev);
12730         int err;
12731
12732         pci_restore_state(tp->pdev);
12733
12734         if (!netif_running(dev))
12735                 return 0;
12736
12737         err = tg3_set_power_state(tp, PCI_D0);
12738         if (err)
12739                 return err;
12740
12741         netif_device_attach(dev);
12742
12743         tg3_full_lock(tp, 0);
12744
12745         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
12746         err = tg3_restart_hw(tp, 1);
12747         if (err)
12748                 goto out;
12749
12750         tp->timer.expires = jiffies + tp->timer_offset;
12751         add_timer(&tp->timer);
12752
12753         tg3_netif_start(tp);
12754
12755 out:
12756         tg3_full_unlock(tp);
12757
12758         return err;
12759 }
12760
12761 static struct pci_driver tg3_driver = {
12762         .name           = DRV_MODULE_NAME,
12763         .id_table       = tg3_pci_tbl,
12764         .probe          = tg3_init_one,
12765         .remove         = __devexit_p(tg3_remove_one),
12766         .suspend        = tg3_suspend,
12767         .resume         = tg3_resume
12768 };
12769
12770 static int __init tg3_init(void)
12771 {
12772         return pci_register_driver(&tg3_driver);
12773 }
12774
12775 static void __exit tg3_cleanup(void)
12776 {
12777         pci_unregister_driver(&tg3_driver);
12778 }
12779
12780 module_init(tg3_init);
12781 module_exit(tg3_cleanup);