#include <linux/etherdevice.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
+#include <linux/inet_lro.h>
#include <linux/ip.h>
#include <linux/inet.h>
#include <linux/in.h>
#include <linux/io.h>
#include <linux/log2.h>
#include <net/checksum.h>
+#include <net/ip.h>
+#include <net/tcp.h>
#include <asm/byteorder.h>
#include <asm/io.h>
#include <asm/processor.h>
#include "myri10ge_mcp.h"
#include "myri10ge_mcp_gen_header.h"
-#define MYRI10GE_VERSION_STR "1.3.2-1.269"
+#define MYRI10GE_VERSION_STR "1.3.2-1.287"
MODULE_DESCRIPTION("Myricom 10G driver (10GbE)");
MODULE_AUTHOR("Maintainer: help@myri.com");
#define MYRI10GE_EEPROM_STRINGS_SIZE 256
#define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2)
+#define MYRI10GE_MAX_LRO_DESCRIPTORS 8
+#define MYRI10GE_LRO_MAX_PKTS 64
#define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff)
#define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff
dma_addr_t bus;
int cnt;
int idx;
+ struct net_lro_mgr lro_mgr;
+ struct net_lro_desc lro_desc[MYRI10GE_MAX_LRO_DESCRIPTORS];
};
struct myri10ge_priv {
int small_bytes;
int big_bytes;
struct net_device *dev;
+ struct napi_struct napi;
struct net_device_stats stats;
u8 __iomem *sram;
int sram_size;
dma_addr_t fw_stats_bus;
struct pci_dev *pdev;
int msi_enabled;
- __be32 link_state;
+ u32 link_state;
unsigned int rdma_tags_available;
int intr_coal_delay;
__be32 __iomem *intr_coal_delay_ptr;
unsigned long serial_number;
int vendor_specific_offset;
int fw_multicast_support;
+ unsigned long features;
+ u32 max_tso6;
u32 read_dma;
u32 write_dma;
u32 read_write_dma;
module_param(myri10ge_debug, int, 0);
MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)");
+static int myri10ge_lro = 1;
+module_param(myri10ge_lro, int, S_IRUGO);
+MODULE_PARM_DESC(myri10ge_lro, "Enable large receive offload\n");
+
+static int myri10ge_lro_max_pkts = MYRI10GE_LRO_MAX_PKTS;
+module_param(myri10ge_lro_max_pkts, int, S_IRUGO);
+MODULE_PARM_DESC(myri10ge_lro, "Number of LRO packets to be aggregated\n");
+
static int myri10ge_fill_thresh = 256;
module_param(myri10ge_fill_thresh, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed\n");
#define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8)
static void myri10ge_set_multicast_list(struct net_device *dev);
+static int myri10ge_sw_tso(struct sk_buff *skb, struct net_device *dev);
static inline void put_be32(__be32 val, __be32 __iomem * p)
{
int status;
/* find running firmware header */
- hdr_offset = ntohl(__raw_readl(mgp->sram + MCP_HEADER_PTR_OFFSET));
+ hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET));
if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) {
dev_err(dev, "Running firmware has bad header offset (%d)\n",
__be32 buf[16];
u32 dma_low, dma_high, size;
int status, i;
+ struct myri10ge_cmd cmd;
size = 0;
status = myri10ge_load_hotplug_firmware(mgp, &size);
dev_info(&mgp->pdev->dev, "handoff confirmed\n");
myri10ge_dummy_rdma(mgp, 1);
+ /* probe for IPv6 TSO support */
+ mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO;
+ status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE,
+ &cmd, 0);
+ if (status == 0) {
+ mgp->max_tso6 = cmd.data0;
+ mgp->features |= NETIF_F_TSO6;
+ }
return 0;
}
remainder -= MYRI10GE_ALLOC_SIZE;
}
+ if (mgp->csum_flag && myri10ge_lro) {
+ rx_frags[0].page_offset += MXGEFW_PAD;
+ rx_frags[0].size -= MXGEFW_PAD;
+ len -= MXGEFW_PAD;
+ lro_receive_frags(&mgp->rx_done.lro_mgr, rx_frags,
+ len, len,
+ /* opaque, will come back in get_frag_header */
+ (void *)(__force unsigned long)csum,
+ csum);
+ return 1;
+ }
+
hlen = MYRI10GE_HLEN > len ? len : MYRI10GE_HLEN;
- /* allocate an skb to attach the page(s) to. */
+ /* allocate an skb to attach the page(s) to. This is done
+ * after trying LRO, so as to avoid skb allocation overheads */
skb = netdev_alloc_skb(dev, MYRI10GE_HLEN + 16);
if (unlikely(skb == NULL)) {
}
}
-static inline void myri10ge_clean_rx_done(struct myri10ge_priv *mgp, int *limit)
+static inline int myri10ge_clean_rx_done(struct myri10ge_priv *mgp, int budget)
{
struct myri10ge_rx_done *rx_done = &mgp->rx_done;
unsigned long rx_bytes = 0;
int idx = rx_done->idx;
int cnt = rx_done->cnt;
+ int work_done = 0;
u16 length;
__wsum checksum;
- while (rx_done->entry[idx].length != 0 && *limit != 0) {
+ while (rx_done->entry[idx].length != 0 && work_done < budget) {
length = ntohs(rx_done->entry[idx].length);
rx_done->entry[idx].length = 0;
checksum = csum_unfold(rx_done->entry[idx].checksum);
rx_bytes += rx_ok * (unsigned long)length;
cnt++;
idx = cnt & (myri10ge_max_intr_slots - 1);
-
- /* limit potential for livelock by only handling a
- * limited number of frames. */
- (*limit)--;
+ work_done++;
}
rx_done->idx = idx;
rx_done->cnt = cnt;
mgp->stats.rx_packets += rx_packets;
mgp->stats.rx_bytes += rx_bytes;
+ if (myri10ge_lro)
+ lro_flush_all(&rx_done->lro_mgr);
+
/* restock receive rings if needed */
if (mgp->rx_small.fill_cnt - mgp->rx_small.cnt < myri10ge_fill_thresh)
myri10ge_alloc_rx_pages(mgp, &mgp->rx_small,
if (mgp->rx_big.fill_cnt - mgp->rx_big.cnt < myri10ge_fill_thresh)
myri10ge_alloc_rx_pages(mgp, &mgp->rx_big, mgp->big_bytes, 0);
+ return work_done;
}
static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp)
}
}
-static int myri10ge_poll(struct net_device *netdev, int *budget)
+static int myri10ge_poll(struct napi_struct *napi, int budget)
{
- struct myri10ge_priv *mgp = netdev_priv(netdev);
- struct myri10ge_rx_done *rx_done = &mgp->rx_done;
- int limit, orig_limit, work_done;
+ struct myri10ge_priv *mgp =
+ container_of(napi, struct myri10ge_priv, napi);
+ struct net_device *netdev = mgp->dev;
+ int work_done;
/* process as many rx events as NAPI will allow */
- limit = min(*budget, netdev->quota);
- orig_limit = limit;
- myri10ge_clean_rx_done(mgp, &limit);
- work_done = orig_limit - limit;
- *budget -= work_done;
- netdev->quota -= work_done;
-
- if (rx_done->entry[rx_done->idx].length == 0 || !netif_running(netdev)) {
- netif_rx_complete(netdev);
+ work_done = myri10ge_clean_rx_done(mgp, budget);
+
+ if (work_done < budget) {
+ netif_rx_complete(netdev, napi);
put_be32(htonl(3), mgp->irq_claim);
- return 0;
}
- return 1;
+ return work_done;
}
static irqreturn_t myri10ge_intr(int irq, void *arg)
/* low bit indicates receives are present, so schedule
* napi poll handler */
if (stats->valid & 1)
- netif_rx_schedule(mgp->dev);
+ netif_rx_schedule(mgp->dev, &mgp->napi);
if (!mgp->msi_enabled) {
put_be32(0, mgp->irq_deassert);
return 0;
}
+static int myri10ge_set_tso(struct net_device *netdev, u32 tso_enabled)
+{
+ struct myri10ge_priv *mgp = netdev_priv(netdev);
+ unsigned long flags = mgp->features & (NETIF_F_TSO6 | NETIF_F_TSO);
+
+ if (tso_enabled)
+ netdev->features |= flags;
+ else
+ netdev->features &= ~flags;
+ return 0;
+}
+
static const char myri10ge_gstrings_stats[][ETH_GSTRING_LEN] = {
"rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors",
"tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions",
"dropped_pause", "dropped_bad_phy", "dropped_bad_crc32",
"dropped_unicast_filtered", "dropped_multicast_filtered",
"dropped_runt", "dropped_overrun", "dropped_no_small_buffer",
- "dropped_no_big_buffer"
+ "dropped_no_big_buffer", "LRO aggregated", "LRO flushed",
+ "LRO avg aggr", "LRO no_desc"
};
#define MYRI10GE_NET_STATS_LEN 21
-#define MYRI10GE_STATS_LEN sizeof(myri10ge_gstrings_stats) / ETH_GSTRING_LEN
+#define MYRI10GE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_stats)
static void
myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data)
}
}
-static int myri10ge_get_stats_count(struct net_device *netdev)
+static int myri10ge_get_sset_count(struct net_device *netdev, int sset)
{
- return MYRI10GE_STATS_LEN;
+ switch (sset) {
+ case ETH_SS_STATS:
+ return MYRI10GE_STATS_LEN;
+ default:
+ return -EOPNOTSUPP;
+ }
}
static void
data[i++] = (unsigned int)ntohl(mgp->fw_stats->dropped_overrun);
data[i++] = (unsigned int)ntohl(mgp->fw_stats->dropped_no_small_buffer);
data[i++] = (unsigned int)ntohl(mgp->fw_stats->dropped_no_big_buffer);
+ data[i++] = mgp->rx_done.lro_mgr.stats.aggregated;
+ data[i++] = mgp->rx_done.lro_mgr.stats.flushed;
+ if (mgp->rx_done.lro_mgr.stats.flushed)
+ data[i++] = mgp->rx_done.lro_mgr.stats.aggregated /
+ mgp->rx_done.lro_mgr.stats.flushed;
+ else
+ data[i++] = 0;
+ data[i++] = mgp->rx_done.lro_mgr.stats.no_desc;
}
static void myri10ge_set_msglevel(struct net_device *netdev, u32 value)
.get_ringparam = myri10ge_get_ringparam,
.get_rx_csum = myri10ge_get_rx_csum,
.set_rx_csum = myri10ge_set_rx_csum,
- .get_tx_csum = ethtool_op_get_tx_csum,
.set_tx_csum = ethtool_op_set_tx_hw_csum,
- .get_sg = ethtool_op_get_sg,
.set_sg = ethtool_op_set_sg,
- .get_tso = ethtool_op_get_tso,
- .set_tso = ethtool_op_set_tso,
+ .set_tso = myri10ge_set_tso,
.get_link = ethtool_op_get_link,
.get_strings = myri10ge_get_strings,
- .get_stats_count = myri10ge_get_stats_count,
+ .get_sset_count = myri10ge_get_sset_count,
.get_ethtool_stats = myri10ge_get_ethtool_stats,
.set_msglevel = myri10ge_set_msglevel,
.get_msglevel = myri10ge_get_msglevel
pci_disable_msi(pdev);
}
+static int
+myri10ge_get_frag_header(struct skb_frag_struct *frag, void **mac_hdr,
+ void **ip_hdr, void **tcpudp_hdr,
+ u64 * hdr_flags, void *priv)
+{
+ struct ethhdr *eh;
+ struct vlan_ethhdr *veh;
+ struct iphdr *iph;
+ u8 *va = page_address(frag->page) + frag->page_offset;
+ unsigned long ll_hlen;
+ /* passed opaque through lro_receive_frags() */
+ __wsum csum = (__force __wsum) (unsigned long)priv;
+
+ /* find the mac header, aborting if not IPv4 */
+
+ eh = (struct ethhdr *)va;
+ *mac_hdr = eh;
+ ll_hlen = ETH_HLEN;
+ if (eh->h_proto != htons(ETH_P_IP)) {
+ if (eh->h_proto == htons(ETH_P_8021Q)) {
+ veh = (struct vlan_ethhdr *)va;
+ if (veh->h_vlan_encapsulated_proto != htons(ETH_P_IP))
+ return -1;
+
+ ll_hlen += VLAN_HLEN;
+
+ /*
+ * HW checksum starts ETH_HLEN bytes into
+ * frame, so we must subtract off the VLAN
+ * header's checksum before csum can be used
+ */
+ csum = csum_sub(csum, csum_partial(va + ETH_HLEN,
+ VLAN_HLEN, 0));
+ } else {
+ return -1;
+ }
+ }
+ *hdr_flags = LRO_IPV4;
+
+ iph = (struct iphdr *)(va + ll_hlen);
+ *ip_hdr = iph;
+ if (iph->protocol != IPPROTO_TCP)
+ return -1;
+ *hdr_flags |= LRO_TCP;
+ *tcpudp_hdr = (u8 *) (*ip_hdr) + (iph->ihl << 2);
+
+ /* verify the IP checksum */
+ if (unlikely(ip_fast_csum((u8 *) iph, iph->ihl)))
+ return -1;
+
+ /* verify the checksum */
+ if (unlikely(csum_tcpudp_magic(iph->saddr, iph->daddr,
+ ntohs(iph->tot_len) - (iph->ihl << 2),
+ IPPROTO_TCP, csum)))
+ return -1;
+
+ return 0;
+}
+
static int myri10ge_open(struct net_device *dev)
{
struct myri10ge_priv *mgp;
struct myri10ge_cmd cmd;
+ struct net_lro_mgr *lro_mgr;
int status, big_pow2;
mgp = netdev_priv(dev);
goto abort_with_rings;
}
- mgp->link_state = htonl(~0U);
+ mgp->link_state = ~0U;
mgp->rdma_tags_available = 15;
- netif_poll_enable(mgp->dev); /* must happen prior to any irq */
+ lro_mgr = &mgp->rx_done.lro_mgr;
+ lro_mgr->dev = dev;
+ lro_mgr->features = LRO_F_NAPI;
+ lro_mgr->ip_summed = CHECKSUM_COMPLETE;
+ lro_mgr->ip_summed_aggr = CHECKSUM_UNNECESSARY;
+ lro_mgr->max_desc = MYRI10GE_MAX_LRO_DESCRIPTORS;
+ lro_mgr->lro_arr = mgp->rx_done.lro_desc;
+ lro_mgr->get_frag_header = myri10ge_get_frag_header;
+ lro_mgr->max_aggr = myri10ge_lro_max_pkts;
+ lro_mgr->frag_align_pad = 2;
+ if (lro_mgr->max_aggr > MAX_SKB_FRAGS)
+ lro_mgr->max_aggr = MAX_SKB_FRAGS;
+
+ napi_enable(&mgp->napi); /* must happen prior to any irq */
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0);
if (status) {
del_timer_sync(&mgp->watchdog_timer);
mgp->running = MYRI10GE_ETH_STOPPING;
- netif_poll_disable(mgp->dev);
+ napi_disable(&mgp->napi);
netif_carrier_off(dev);
netif_stop_queue(dev);
old_down_cnt = mgp->down_cnt;
pseudo_hdr_offset = cksum_offset + skb->csum_offset;
/* If the headers are excessively large, then we must
* fall back to a software checksum */
- if (unlikely(cksum_offset > 255 || pseudo_hdr_offset > 127)) {
+ if (unlikely(!mss && (cksum_offset > 255 ||
+ pseudo_hdr_offset > 127))) {
if (skb_checksum_help(skb))
goto drop;
cksum_offset = 0;
/* negative cum_len signifies to the
* send loop that we are still in the
* header portion of the TSO packet.
- * TSO header must be at most 134 bytes long */
+ * TSO header can be at most 1KB long */
cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb));
+ /* for IPv6 TSO, the checksum offset stores the
+ * TCP header length, to save the firmware from
+ * the need to parse the headers */
+ if (skb_is_gso_v6(skb)) {
+ cksum_offset = tcp_hdrlen(skb);
+ /* Can only handle headers <= max_tso6 long */
+ if (unlikely(-cum_len > mgp->max_tso6))
+ return myri10ge_sw_tso(skb, dev);
+ }
/* for TSO, pseudo_hdr_offset holds mss.
* The firmware figures out where to put
* the checksum by parsing the header. */
req++;
count++;
rdma_count++;
- if (unlikely(cksum_offset > seglen))
- cksum_offset -= seglen;
- else
- cksum_offset = 0;
+ if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) {
+ if (unlikely(cksum_offset > seglen))
+ cksum_offset -= seglen;
+ else
+ cksum_offset = 0;
+ }
}
if (frag_idx == frag_cnt)
break;
}
+static int myri10ge_sw_tso(struct sk_buff *skb, struct net_device *dev)
+{
+ struct sk_buff *segs, *curr;
+ struct myri10ge_priv *mgp = dev->priv;
+ int status;
+
+ segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6);
+ if (IS_ERR(segs))
+ goto drop;
+
+ while (segs) {
+ curr = segs;
+ segs = segs->next;
+ curr->next = NULL;
+ status = myri10ge_xmit(curr, dev);
+ if (status != 0) {
+ dev_kfree_skb_any(curr);
+ if (segs != NULL) {
+ curr = segs;
+ segs = segs->next;
+ curr->next = NULL;
+ dev_kfree_skb_any(segs);
+ }
+ goto drop;
+ }
+ }
+ dev_kfree_skb_any(skb);
+ return 0;
+
+drop:
+ dev_kfree_skb_any(skb);
+ mgp->stats.tx_dropped += 1;
+ return 0;
+}
+
static struct net_device_stats *myri10ge_get_stats(struct net_device *dev)
{
struct myri10ge_priv *mgp = netdev_priv(dev);
struct dev_mc_list *mc_list;
__be32 data[2] = { 0, 0 };
int err;
+ DECLARE_MAC_BUF(mac);
mgp = netdev_priv(dev);
/* can be called from atomic contexts,
printk(KERN_ERR "myri10ge: %s: Failed "
"MXGEFW_JOIN_MULTICAST_GROUP, error status:"
"%d\t", dev->name, err);
- printk(KERN_ERR "MAC %02x:%02x:%02x:%02x:%02x:%02x\n",
- ((unsigned char *)&mc_list->dmi_addr)[0],
- ((unsigned char *)&mc_list->dmi_addr)[1],
- ((unsigned char *)&mc_list->dmi_addr)[2],
- ((unsigned char *)&mc_list->dmi_addr)[3],
- ((unsigned char *)&mc_list->dmi_addr)[4],
- ((unsigned char *)&mc_list->dmi_addr)[5]
- );
+ printk(KERN_ERR "MAC %s\n",
+ print_mac(mac, mc_list->dmi_addr));
goto abort;
}
}
}
#ifdef CONFIG_PM
-
static int myri10ge_suspend(struct pci_dev *pdev, pm_message_t state)
{
struct myri10ge_priv *mgp;
return -EIO;
}
-
#endif /* CONFIG_PM */
static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp)
SET_NETDEV_DEV(netdev, &pdev->dev);
mgp = netdev_priv(netdev);
- memset(mgp, 0, sizeof(*mgp));
mgp->dev = netdev;
+ netif_napi_add(netdev, &mgp->napi, myri10ge_poll, myri10ge_napi_weight);
mgp->pdev = pdev;
mgp->csum_flag = MXGEFW_FLAGS_CKSUM;
mgp->pause = myri10ge_flow_control;
if (status != 0) {
dac_enabled = 0;
dev_err(&pdev->dev,
- "64-bit pci address mask was refused, trying 32-bit");
+ "64-bit pci address mask was refused, "
+ "trying 32-bit\n");
status = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
}
if (status != 0) {
netdev->change_mtu = myri10ge_change_mtu;
netdev->set_multicast_list = myri10ge_set_multicast_list;
netdev->set_mac_address = myri10ge_set_mac_address;
- netdev->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO;
+ netdev->features = mgp->features;
if (dac_enabled)
netdev->features |= NETIF_F_HIGHDMA;
- netdev->poll = myri10ge_poll;
- netdev->weight = myri10ge_napi_weight;
/* make sure we can get an irq, and that MSI can be
* setup (if available). Also ensure netdev->irq
}
#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008
+#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009
static struct pci_device_id myri10ge_pci_tbl[] = {
{PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)},
+ {PCI_DEVICE
+ (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)},
{0},
};
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff