tree-wide: fix assorted typos all over the place
[safe/jmp/linux-2.6] / drivers / net / cxgb3 / sge.c
index 90f6f82..56ba872 100644 (file)
@@ -50,6 +50,7 @@
 #define SGE_RX_COPY_THRES  256
 #define SGE_RX_PULL_LEN    128
 
+#define SGE_PG_RSVD SMP_CACHE_BYTES
 /*
  * Page chunk size for FL0 buffers if FL0 is to be populated with page chunks.
  * It must be a divisor of PAGE_SIZE.  If set to 0 FL0 will use sk_buffs
  */
 #define FL0_PG_CHUNK_SIZE  2048
 #define FL0_PG_ORDER 0
+#define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER)
 #define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
 #define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
+#define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER)
 
 #define SGE_RX_DROP_THRES 16
+#define RX_RECLAIM_PERIOD (HZ/4)
 
 /*
  * Max number of Rx buffers we replenish at a time.
@@ -71,6 +75,8 @@
  * frequently as Tx buffers are usually reclaimed by new Tx packets.
  */
 #define TX_RECLAIM_PERIOD (HZ / 4)
+#define TX_RECLAIM_TIMER_CHUNK 64U
+#define TX_RECLAIM_CHUNK 16U
 
 /* WR size in bytes */
 #define WR_LEN (WR_FLITS * 8)
@@ -308,21 +314,25 @@ static void free_tx_desc(struct adapter *adapter, struct sge_txq *q,
  *     reclaim_completed_tx - reclaims completed Tx descriptors
  *     @adapter: the adapter
  *     @q: the Tx queue to reclaim completed descriptors from
+ *     @chunk: maximum number of descriptors to reclaim
  *
  *     Reclaims Tx descriptors that the SGE has indicated it has processed,
  *     and frees the associated buffers if possible.  Called with the Tx
  *     queue's lock held.
  */
-static inline void reclaim_completed_tx(struct adapter *adapter,
-                                       struct sge_txq *q)
+static inline unsigned int reclaim_completed_tx(struct adapter *adapter,
+                                               struct sge_txq *q,
+                                               unsigned int chunk)
 {
        unsigned int reclaim = q->processed - q->cleaned;
 
+       reclaim = min(chunk, reclaim);
        if (reclaim) {
                free_tx_desc(adapter, q, reclaim);
                q->cleaned += reclaim;
                q->in_use -= reclaim;
        }
+       return q->processed - q->cleaned;
 }
 
 /**
@@ -338,13 +348,21 @@ static inline int should_restart_tx(const struct sge_txq *q)
        return q->in_use - r < (q->size >> 1);
 }
 
-static void clear_rx_desc(const struct sge_fl *q, struct rx_sw_desc *d)
+static void clear_rx_desc(struct pci_dev *pdev, const struct sge_fl *q,
+                         struct rx_sw_desc *d)
 {
-       if (q->use_pages) {
-               if (d->pg_chunk.page)
-                       put_page(d->pg_chunk.page);
+       if (q->use_pages && d->pg_chunk.page) {
+               (*d->pg_chunk.p_cnt)--;
+               if (!*d->pg_chunk.p_cnt)
+                       pci_unmap_page(pdev,
+                                      d->pg_chunk.mapping,
+                                      q->alloc_size, PCI_DMA_FROMDEVICE);
+
+               put_page(d->pg_chunk.page);
                d->pg_chunk.page = NULL;
        } else {
+               pci_unmap_single(pdev, pci_unmap_addr(d, dma_addr),
+                                q->buf_size, PCI_DMA_FROMDEVICE);
                kfree_skb(d->skb);
                d->skb = NULL;
        }
@@ -365,9 +383,8 @@ static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q)
        while (q->credits--) {
                struct rx_sw_desc *d = &q->sdesc[cidx];
 
-               pci_unmap_single(pdev, pci_unmap_addr(d, dma_addr),
-                                q->buf_size, PCI_DMA_FROMDEVICE);
-               clear_rx_desc(q, d);
+
+               clear_rx_desc(pdev, q, d);
                if (++cidx == q->size)
                        cidx = 0;
        }
@@ -410,18 +427,39 @@ static inline int add_one_rx_buf(void *va, unsigned int len,
        return 0;
 }
 
-static int alloc_pg_chunk(struct sge_fl *q, struct rx_sw_desc *sd, gfp_t gfp,
+static inline int add_one_rx_chunk(dma_addr_t mapping, struct rx_desc *d,
+                                  unsigned int gen)
+{
+       d->addr_lo = cpu_to_be32(mapping);
+       d->addr_hi = cpu_to_be32((u64) mapping >> 32);
+       wmb();
+       d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
+       d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
+       return 0;
+}
+
+static int alloc_pg_chunk(struct adapter *adapter, struct sge_fl *q,
+                         struct rx_sw_desc *sd, gfp_t gfp,
                          unsigned int order)
 {
        if (!q->pg_chunk.page) {
+               dma_addr_t mapping;
+
                q->pg_chunk.page = alloc_pages(gfp, order);
                if (unlikely(!q->pg_chunk.page))
                        return -ENOMEM;
                q->pg_chunk.va = page_address(q->pg_chunk.page);
+               q->pg_chunk.p_cnt = q->pg_chunk.va + (PAGE_SIZE << order) -
+                                   SGE_PG_RSVD;
                q->pg_chunk.offset = 0;
+               mapping = pci_map_page(adapter->pdev, q->pg_chunk.page,
+                                      0, q->alloc_size, PCI_DMA_FROMDEVICE);
+               q->pg_chunk.mapping = mapping;
        }
        sd->pg_chunk = q->pg_chunk;
 
+       prefetch(sd->pg_chunk.p_cnt);
+
        q->pg_chunk.offset += q->buf_size;
        if (q->pg_chunk.offset == (PAGE_SIZE << order))
                q->pg_chunk.page = NULL;
@@ -429,6 +467,12 @@ static int alloc_pg_chunk(struct sge_fl *q, struct rx_sw_desc *sd, gfp_t gfp,
                q->pg_chunk.va += q->buf_size;
                get_page(q->pg_chunk.page);
        }
+
+       if (sd->pg_chunk.offset == 0)
+               *sd->pg_chunk.p_cnt = 1;
+       else
+               *sd->pg_chunk.p_cnt += 1;
+
        return 0;
 }
 
@@ -453,35 +497,42 @@ static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
  */
 static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp)
 {
-       void *buf_start;
        struct rx_sw_desc *sd = &q->sdesc[q->pidx];
        struct rx_desc *d = &q->desc[q->pidx];
        unsigned int count = 0;
 
        while (n--) {
+               dma_addr_t mapping;
                int err;
 
                if (q->use_pages) {
-                       if (unlikely(alloc_pg_chunk(q, sd, gfp, q->order))) {
+                       if (unlikely(alloc_pg_chunk(adap, q, sd, gfp,
+                                                   q->order))) {
 nomem:                         q->alloc_failed++;
                                break;
                        }
-                       buf_start = sd->pg_chunk.va;
+                       mapping = sd->pg_chunk.mapping + sd->pg_chunk.offset;
+                       pci_unmap_addr_set(sd, dma_addr, mapping);
+
+                       add_one_rx_chunk(mapping, d, q->gen);
+                       pci_dma_sync_single_for_device(adap->pdev, mapping,
+                                               q->buf_size - SGE_PG_RSVD,
+                                               PCI_DMA_FROMDEVICE);
                } else {
-                       struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
+                       void *buf_start;
 
+                       struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
                        if (!skb)
                                goto nomem;
 
                        sd->skb = skb;
                        buf_start = skb->data;
-               }
-
-               err = add_one_rx_buf(buf_start, q->buf_size, d, sd, q->gen,
-                                    adap->pdev);
-               if (unlikely(err)) {
-                       clear_rx_desc(q, sd);
-                       break;
+                       err = add_one_rx_buf(buf_start, q->buf_size, d, sd,
+                                            q->gen, adap->pdev);
+                       if (unlikely(err)) {
+                               clear_rx_desc(adap->pdev, q, sd);
+                               break;
+                       }
                }
 
                d++;
@@ -601,7 +652,9 @@ static void t3_reset_qset(struct sge_qset *q)
        memset(q->txq, 0, sizeof(struct sge_txq) * SGE_TXQ_PER_SET);
        q->txq_stopped = 0;
        q->tx_reclaim_timer.function = NULL; /* for t3_stop_sge_timers() */
-       q->lro_frag_tbl.nr_frags = q->lro_frag_tbl.len = 0;
+       q->rx_reclaim_timer.function = NULL;
+       q->nomem = 0;
+       napi_free_frags(&q->napi);
 }
 
 
@@ -787,19 +840,19 @@ static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
        struct sk_buff *newskb, *skb;
        struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
 
-       newskb = skb = q->pg_skb;
+       dma_addr_t dma_addr = pci_unmap_addr(sd, dma_addr);
 
+       newskb = skb = q->pg_skb;
        if (!skb && (len <= SGE_RX_COPY_THRES)) {
                newskb = alloc_skb(len, GFP_ATOMIC);
                if (likely(newskb != NULL)) {
                        __skb_put(newskb, len);
-                       pci_dma_sync_single_for_cpu(adap->pdev,
-                                           pci_unmap_addr(sd, dma_addr), len,
+                       pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
                                            PCI_DMA_FROMDEVICE);
                        memcpy(newskb->data, sd->pg_chunk.va, len);
-                       pci_dma_sync_single_for_device(adap->pdev,
-                                           pci_unmap_addr(sd, dma_addr), len,
-                                           PCI_DMA_FROMDEVICE);
+                       pci_dma_sync_single_for_device(adap->pdev, dma_addr,
+                                                      len,
+                                                      PCI_DMA_FROMDEVICE);
                } else if (!drop_thres)
                        return NULL;
 recycle:
@@ -812,16 +865,25 @@ recycle:
        if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres)))
                goto recycle;
 
+       prefetch(sd->pg_chunk.p_cnt);
+
        if (!skb)
                newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
+
        if (unlikely(!newskb)) {
                if (!drop_thres)
                        return NULL;
                goto recycle;
        }
 
-       pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr),
-                        fl->buf_size, PCI_DMA_FROMDEVICE);
+       pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
+                                   PCI_DMA_FROMDEVICE);
+       (*sd->pg_chunk.p_cnt)--;
+       if (!*sd->pg_chunk.p_cnt)
+               pci_unmap_page(adap->pdev,
+                              sd->pg_chunk.mapping,
+                              fl->alloc_size,
+                              PCI_DMA_FROMDEVICE);
        if (!skb) {
                __skb_put(newskb, SGE_RX_PULL_LEN);
                memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
@@ -1081,7 +1143,7 @@ static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb,
        struct tx_desc *d = &q->desc[pidx];
        struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d;
 
-       cpl->len = htonl(skb->len | 0x80000000);
+       cpl->len = htonl(skb->len);
        cntrl = V_TXPKT_INTF(pi->port_id);
 
        if (vlan_tx_tag_present(skb) && pi->vlan_grp)
@@ -1154,7 +1216,7 @@ static inline void t3_stop_tx_queue(struct netdev_queue *txq,
  *
  *     Add a packet to an SGE Tx queue.  Runs with softirqs disabled.
  */
-int t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
+netdev_tx_t t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
 {
        int qidx;
        unsigned int ndesc, pidx, credits, gen, compl;
@@ -1178,8 +1240,7 @@ int t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
        q = &qs->txq[TXQ_ETH];
        txq = netdev_get_tx_queue(dev, qidx);
 
-       spin_lock(&q->lock);
-       reclaim_completed_tx(adap, q);
+       reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
 
        credits = q->size - q->in_use;
        ndesc = calc_tx_descs(skb);
@@ -1189,7 +1250,6 @@ int t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
                dev_err(&adap->pdev->dev,
                        "%s: Tx ring %u full while queue awake!\n",
                        dev->name, q->cntxt_id & 7);
-               spin_unlock(&q->lock);
                return NETDEV_TX_BUSY;
        }
 
@@ -1223,12 +1283,9 @@ int t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
        if (vlan_tx_tag_present(skb) && pi->vlan_grp)
                qs->port_stats[SGE_PSTAT_VLANINS]++;
 
-       dev->trans_start = jiffies;
-       spin_unlock(&q->lock);
-
        /*
         * We do not use Tx completion interrupts to free DMAd Tx packets.
-        * This is good for performamce but means that we rely on new Tx
+        * This is good for performance but means that we rely on new Tx
         * packets arriving to run the destructors of completed packets,
         * which open up space in their sockets' send queues.  Sometimes
         * we do not get such new packets causing Tx to stall.  A single
@@ -1588,7 +1645,7 @@ static int ofld_xmit(struct adapter *adap, struct sge_txq *q,
        unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen;
 
        spin_lock(&q->lock);
-      again:reclaim_completed_tx(adap, q);
+again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
 
        ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD);
        if (unlikely(ret)) {
@@ -1630,7 +1687,7 @@ static void restart_offloadq(unsigned long data)
        struct adapter *adap = pi->adapter;
 
        spin_lock(&q->lock);
-      again:reclaim_completed_tx(adap, q);
+again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
 
        while ((skb = skb_peek(&q->sendq)) != NULL) {
                unsigned int gen, pidx;
@@ -1950,8 +2007,8 @@ static void rx_eth(struct adapter *adap, struct sge_rspq *rq,
        skb_pull(skb, sizeof(*p) + pad);
        skb->protocol = eth_type_trans(skb, adap->port[p->iff]);
        pi = netdev_priv(skb->dev);
-       if ((pi->rx_offload & T3_RX_CSUM) && p->csum_valid && p->csum == htons(0xffff) &&
-           !p->fragment) {
+       if ((pi->rx_offload & T3_RX_CSUM) && p->csum_valid &&
+           p->csum == htons(0xffff) && !p->fragment) {
                qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
                skb->ip_summed = CHECKSUM_UNNECESSARY;
        } else
@@ -2012,35 +2069,63 @@ static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
                         struct sge_fl *fl, int len, int complete)
 {
        struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
+       struct sk_buff *skb = NULL;
        struct cpl_rx_pkt *cpl;
-       struct skb_frag_struct *rx_frag = qs->lro_frag_tbl.frags;
-       int nr_frags = qs->lro_frag_tbl.nr_frags;
-       int frag_len = qs->lro_frag_tbl.len;
+       struct skb_frag_struct *rx_frag;
+       int nr_frags;
        int offset = 0;
 
-       if (!nr_frags) {
-               offset = 2 + sizeof(struct cpl_rx_pkt);
-               qs->lro_va = cpl = sd->pg_chunk.va + 2;
+       if (!qs->nomem) {
+               skb = napi_get_frags(&qs->napi);
+               qs->nomem = !skb;
        }
 
        fl->credits--;
 
+       pci_dma_sync_single_for_cpu(adap->pdev,
+                                   pci_unmap_addr(sd, dma_addr),
+                                   fl->buf_size - SGE_PG_RSVD,
+                                   PCI_DMA_FROMDEVICE);
+
+       (*sd->pg_chunk.p_cnt)--;
+       if (!*sd->pg_chunk.p_cnt)
+               pci_unmap_page(adap->pdev,
+                              sd->pg_chunk.mapping,
+                              fl->alloc_size,
+                              PCI_DMA_FROMDEVICE);
+
+       if (!skb) {
+               put_page(sd->pg_chunk.page);
+               if (complete)
+                       qs->nomem = 0;
+               return;
+       }
+
+       rx_frag = skb_shinfo(skb)->frags;
+       nr_frags = skb_shinfo(skb)->nr_frags;
+
+       if (!nr_frags) {
+               offset = 2 + sizeof(struct cpl_rx_pkt);
+               qs->lro_va = sd->pg_chunk.va + 2;
+       }
        len -= offset;
-       pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr),
-                        fl->buf_size, PCI_DMA_FROMDEVICE);
+
+       prefetch(qs->lro_va);
 
        rx_frag += nr_frags;
        rx_frag->page = sd->pg_chunk.page;
        rx_frag->page_offset = sd->pg_chunk.offset + offset;
        rx_frag->size = len;
-       frag_len += len;
-       qs->lro_frag_tbl.nr_frags++;
-       qs->lro_frag_tbl.len = frag_len;
+
+       skb->len += len;
+       skb->data_len += len;
+       skb->truesize += len;
+       skb_shinfo(skb)->nr_frags++;
 
        if (!complete)
                return;
 
-       qs->lro_frag_tbl.ip_summed = CHECKSUM_UNNECESSARY;
+       skb->ip_summed = CHECKSUM_UNNECESSARY;
        cpl = qs->lro_va;
 
        if (unlikely(cpl->vlan_valid)) {
@@ -2049,15 +2134,11 @@ static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
                struct vlan_group *grp = pi->vlan_grp;
 
                if (likely(grp != NULL)) {
-                       vlan_gro_frags(&qs->napi, grp, ntohs(cpl->vlan),
-                                      &qs->lro_frag_tbl);
-                       goto out;
+                       vlan_gro_frags(&qs->napi, grp, ntohs(cpl->vlan));
+                       return;
                }
        }
-       napi_gro_frags(&qs->napi, &qs->lro_frag_tbl);
-
-out:
-       qs->lro_frag_tbl.nr_frags = qs->lro_frag_tbl.len = 0;
+       napi_gro_frags(&qs->napi);
 }
 
 /**
@@ -2715,7 +2796,8 @@ irq_handler_t t3_intr_handler(struct adapter *adap, int polling)
  */
 void t3_sge_err_intr_handler(struct adapter *adapter)
 {
-       unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE);
+       unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE) &
+                                ~F_FLEMPTY;
 
        if (status & SGE_PARERR)
                CH_ALERT(adapter, "SGE parity error (0x%x)\n",
@@ -2745,13 +2827,13 @@ void t3_sge_err_intr_handler(struct adapter *adapter)
 }
 
 /**
- *     sge_timer_cb - perform periodic maintenance of an SGE qset
+ *     sge_timer_tx - perform periodic maintenance of an SGE qset
  *     @data: the SGE queue set to maintain
  *
  *     Runs periodically from a timer to perform maintenance of an SGE queue
  *     set.  It performs two tasks:
  *
- *     a) Cleans up any completed Tx descriptors that may still be pending.
+ *     Cleans up any completed Tx descriptors that may still be pending.
  *     Normal descriptor cleanup happens when new packets are added to a Tx
  *     queue so this timer is relatively infrequent and does any cleanup only
  *     if the Tx queue has not seen any new packets in a while.  We make a
@@ -2761,51 +2843,88 @@ void t3_sge_err_intr_handler(struct adapter *adapter)
  *     up).  Since control queues use immediate data exclusively we don't
  *     bother cleaning them up here.
  *
- *     b) Replenishes Rx queues that have run out due to memory shortage.
+ */
+static void sge_timer_tx(unsigned long data)
+{
+       struct sge_qset *qs = (struct sge_qset *)data;
+       struct port_info *pi = netdev_priv(qs->netdev);
+       struct adapter *adap = pi->adapter;
+       unsigned int tbd[SGE_TXQ_PER_SET] = {0, 0};
+       unsigned long next_period;
+
+       if (__netif_tx_trylock(qs->tx_q)) {
+                tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],
+                                                     TX_RECLAIM_TIMER_CHUNK);
+               __netif_tx_unlock(qs->tx_q);
+       }
+
+       if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
+               tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD],
+                                                    TX_RECLAIM_TIMER_CHUNK);
+               spin_unlock(&qs->txq[TXQ_OFLD].lock);
+       }
+
+       next_period = TX_RECLAIM_PERIOD >>
+                      (max(tbd[TXQ_ETH], tbd[TXQ_OFLD]) /
+                      TX_RECLAIM_TIMER_CHUNK);
+       mod_timer(&qs->tx_reclaim_timer, jiffies + next_period);
+}
+
+/*
+ *     sge_timer_rx - perform periodic maintenance of an SGE qset
+ *     @data: the SGE queue set to maintain
+ *
+ *     a) Replenishes Rx queues that have run out due to memory shortage.
  *     Normally new Rx buffers are added when existing ones are consumed but
  *     when out of memory a queue can become empty.  We try to add only a few
  *     buffers here, the queue will be replenished fully as these new buffers
  *     are used up if memory shortage has subsided.
+ *
+ *     b) Return coalesced response queue credits in case a response queue is
+ *     starved.
+ *
  */
-static void sge_timer_cb(unsigned long data)
+static void sge_timer_rx(unsigned long data)
 {
        spinlock_t *lock;
        struct sge_qset *qs = (struct sge_qset *)data;
-       struct adapter *adap = qs->adap;
+       struct port_info *pi = netdev_priv(qs->netdev);
+       struct adapter *adap = pi->adapter;
+       u32 status;
 
-       if (spin_trylock(&qs->txq[TXQ_ETH].lock)) {
-               reclaim_completed_tx(adap, &qs->txq[TXQ_ETH]);
-               spin_unlock(&qs->txq[TXQ_ETH].lock);
-       }
-       if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
-               reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD]);
-               spin_unlock(&qs->txq[TXQ_OFLD].lock);
-       }
-       lock = (adap->flags & USING_MSIX) ? &qs->rspq.lock :
-                                           &adap->sge.qs[0].rspq.lock;
-       if (spin_trylock_irq(lock)) {
-               if (!napi_is_scheduled(&qs->napi)) {
-                       u32 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
-
-                       if (qs->fl[0].credits < qs->fl[0].size)
-                               __refill_fl(adap, &qs->fl[0]);
-                       if (qs->fl[1].credits < qs->fl[1].size)
-                               __refill_fl(adap, &qs->fl[1]);
-
-                       if (status & (1 << qs->rspq.cntxt_id)) {
-                               qs->rspq.starved++;
-                               if (qs->rspq.credits) {
-                                       refill_rspq(adap, &qs->rspq, 1);
-                                       qs->rspq.credits--;
-                                       qs->rspq.restarted++;
-                                       t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
-                                                    1 << qs->rspq.cntxt_id);
-                               }
+       lock = adap->params.rev > 0 ?
+              &qs->rspq.lock : &adap->sge.qs[0].rspq.lock;
+
+       if (!spin_trylock_irq(lock))
+               goto out;
+
+       if (napi_is_scheduled(&qs->napi))
+               goto unlock;
+
+       if (adap->params.rev < 4) {
+               status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
+
+               if (status & (1 << qs->rspq.cntxt_id)) {
+                       qs->rspq.starved++;
+                       if (qs->rspq.credits) {
+                               qs->rspq.credits--;
+                               refill_rspq(adap, &qs->rspq, 1);
+                               qs->rspq.restarted++;
+                               t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
+                                            1 << qs->rspq.cntxt_id);
                        }
                }
-               spin_unlock_irq(lock);
        }
-       mod_timer(&qs->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
+
+       if (qs->fl[0].credits < qs->fl[0].size)
+               __refill_fl(adap, &qs->fl[0]);
+       if (qs->fl[1].credits < qs->fl[1].size)
+               __refill_fl(adap, &qs->fl[1]);
+
+unlock:
+       spin_unlock_irq(lock);
+out:
+       mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
 }
 
 /**
@@ -2848,7 +2967,8 @@ int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
        struct sge_qset *q = &adapter->sge.qs[id];
 
        init_qset_cntxt(q, id);
-       setup_timer(&q->tx_reclaim_timer, sge_timer_cb, (unsigned long)q);
+       setup_timer(&q->tx_reclaim_timer, sge_timer_tx, (unsigned long)q);
+       setup_timer(&q->rx_reclaim_timer, sge_timer_rx, (unsigned long)q);
 
        q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size,
                                   sizeof(struct rx_desc),
@@ -2924,21 +3044,23 @@ int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
        q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0;
        q->fl[0].order = FL0_PG_ORDER;
        q->fl[1].order = FL1_PG_ORDER;
+       q->fl[0].alloc_size = FL0_PG_ALLOC_SIZE;
+       q->fl[1].alloc_size = FL1_PG_ALLOC_SIZE;
 
        spin_lock_irq(&adapter->sge.reg_lock);
 
        /* FL threshold comparison uses < */
        ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx,
                                   q->rspq.phys_addr, q->rspq.size,
-                                  q->fl[0].buf_size, 1, 0);
+                                  q->fl[0].buf_size - SGE_PG_RSVD, 1, 0);
        if (ret)
                goto err_unlock;
 
        for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
                ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0,
                                          q->fl[i].phys_addr, q->fl[i].size,
-                                         q->fl[i].buf_size, p->cong_thres, 1,
-                                         0);
+                                         q->fl[i].buf_size - SGE_PG_RSVD,
+                                         p->cong_thres, 1, 0);
                if (ret)
                        goto err_unlock;
        }
@@ -2996,7 +3118,6 @@ int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
        t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
                     V_NEWTIMER(q->rspq.holdoff_tmr));
 
-       mod_timer(&q->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
        return 0;
 
 err_unlock:
@@ -3007,6 +3128,27 @@ err:
 }
 
 /**
+ *      t3_start_sge_timers - start SGE timer call backs
+ *      @adap: the adapter
+ *
+ *      Starts each SGE queue set's timer call back
+ */
+void t3_start_sge_timers(struct adapter *adap)
+{
+       int i;
+
+       for (i = 0; i < SGE_QSETS; ++i) {
+               struct sge_qset *q = &adap->sge.qs[i];
+
+       if (q->tx_reclaim_timer.function)
+               mod_timer(&q->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
+
+       if (q->rx_reclaim_timer.function)
+               mod_timer(&q->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
+       }
+}
+
+/**
  *     t3_stop_sge_timers - stop SGE timer call backs
  *     @adap: the adapter
  *
@@ -3021,6 +3163,8 @@ void t3_stop_sge_timers(struct adapter *adap)
 
                if (q->tx_reclaim_timer.function)
                        del_timer_sync(&q->tx_reclaim_timer);
+               if (q->rx_reclaim_timer.function)
+                       del_timer_sync(&q->rx_reclaim_timer);
        }
 }