+static inline void niu_sync_rx_discard_stats(struct niu *np,
+ struct rx_ring_info *rp,
+ const int limit)
+{
+ /* This elaborate scheme is needed for reading the RX discard
+ * counters, as they are only 16-bit and can overflow quickly,
+ * and because the overflow indication bit is not usable as
+ * the counter value does not wrap, but remains at max value
+ * 0xFFFF.
+ *
+ * In theory and in practice counters can be lost in between
+ * reading nr64() and clearing the counter nw64(). For this
+ * reason, the number of counter clearings nw64() is
+ * limited/reduced though the limit parameter.
+ */
+ int rx_channel = rp->rx_channel;
+ u32 misc, wred;
+
+ /* RXMISC (Receive Miscellaneous Discard Count), covers the
+ * following discard events: IPP (Input Port Process),
+ * FFLP/TCAM, Full RCR (Receive Completion Ring) RBR (Receive
+ * Block Ring) prefetch buffer is empty.
+ */
+ misc = nr64(RXMISC(rx_channel));
+ if (unlikely((misc & RXMISC_COUNT) > limit)) {
+ nw64(RXMISC(rx_channel), 0);
+ rp->rx_errors += misc & RXMISC_COUNT;
+
+ if (unlikely(misc & RXMISC_OFLOW))
+ dev_err(np->device, "rx-%d: Counter overflow "
+ "RXMISC discard\n", rx_channel);
+
+ niudbg(RX_ERR, "%s-rx-%d: MISC drop=%u over=%u\n",
+ np->dev->name, rx_channel, misc, misc-limit);
+ }
+
+ /* WRED (Weighted Random Early Discard) by hardware */
+ wred = nr64(RED_DIS_CNT(rx_channel));
+ if (unlikely((wred & RED_DIS_CNT_COUNT) > limit)) {
+ nw64(RED_DIS_CNT(rx_channel), 0);
+ rp->rx_dropped += wred & RED_DIS_CNT_COUNT;
+
+ if (unlikely(wred & RED_DIS_CNT_OFLOW))
+ dev_err(np->device, "rx-%d: Counter overflow "
+ "WRED discard\n", rx_channel);
+
+ niudbg(RX_ERR, "%s-rx-%d: WRED drop=%u over=%u\n",
+ np->dev->name, rx_channel, wred, wred-limit);
+ }
+}
+
+static int niu_rx_work(struct napi_struct *napi, struct niu *np,
+ struct rx_ring_info *rp, int budget)
+{
+ int qlen, rcr_done = 0, work_done = 0;
+ struct rxdma_mailbox *mbox = rp->mbox;
+ u64 stat;
+
+#if 1
+ stat = nr64(RX_DMA_CTL_STAT(rp->rx_channel));
+ qlen = nr64(RCRSTAT_A(rp->rx_channel)) & RCRSTAT_A_QLEN;
+#else
+ stat = le64_to_cpup(&mbox->rx_dma_ctl_stat);
+ qlen = (le64_to_cpup(&mbox->rcrstat_a) & RCRSTAT_A_QLEN);
+#endif
+ mbox->rx_dma_ctl_stat = 0;
+ mbox->rcrstat_a = 0;
+
+ niudbg(RX_STATUS, "%s: niu_rx_work(chan[%d]), stat[%llx] qlen=%d\n",
+ np->dev->name, rp->rx_channel, (unsigned long long) stat, qlen);
+
+ rcr_done = work_done = 0;
+ qlen = min(qlen, budget);
+ while (work_done < qlen) {
+ rcr_done += niu_process_rx_pkt(napi, np, rp);
+ work_done++;
+ }
+
+ if (rp->rbr_refill_pending >= rp->rbr_kick_thresh) {
+ unsigned int i;
+
+ for (i = 0; i < rp->rbr_refill_pending; i++)
+ niu_rbr_refill(np, rp, GFP_ATOMIC);
+ rp->rbr_refill_pending = 0;
+ }
+
+ stat = (RX_DMA_CTL_STAT_MEX |
+ ((u64)work_done << RX_DMA_CTL_STAT_PKTREAD_SHIFT) |
+ ((u64)rcr_done << RX_DMA_CTL_STAT_PTRREAD_SHIFT));
+
+ nw64(RX_DMA_CTL_STAT(rp->rx_channel), stat);
+
+ /* Only sync discards stats when qlen indicate potential for drops */
+ if (qlen > 10)
+ niu_sync_rx_discard_stats(np, rp, 0x7FFF);
+
+ return work_done;
+}
+
+static int niu_poll_core(struct niu *np, struct niu_ldg *lp, int budget)
+{
+ u64 v0 = lp->v0;
+ u32 tx_vec = (v0 >> 32);
+ u32 rx_vec = (v0 & 0xffffffff);
+ int i, work_done = 0;
+
+ niudbg(INTR, "%s: niu_poll_core() v0[%016llx]\n",
+ np->dev->name, (unsigned long long) v0);
+
+ for (i = 0; i < np->num_tx_rings; i++) {
+ struct tx_ring_info *rp = &np->tx_rings[i];
+ if (tx_vec & (1 << rp->tx_channel))
+ niu_tx_work(np, rp);
+ nw64(LD_IM0(LDN_TXDMA(rp->tx_channel)), 0);
+ }
+
+ for (i = 0; i < np->num_rx_rings; i++) {
+ struct rx_ring_info *rp = &np->rx_rings[i];
+
+ if (rx_vec & (1 << rp->rx_channel)) {
+ int this_work_done;
+
+ this_work_done = niu_rx_work(&lp->napi, np, rp,
+ budget);
+
+ budget -= this_work_done;
+ work_done += this_work_done;
+ }
+ nw64(LD_IM0(LDN_RXDMA(rp->rx_channel)), 0);
+ }
+
+ return work_done;
+}
+
+static int niu_poll(struct napi_struct *napi, int budget)
+{
+ struct niu_ldg *lp = container_of(napi, struct niu_ldg, napi);
+ struct niu *np = lp->np;
+ int work_done;
+
+ work_done = niu_poll_core(np, lp, budget);
+
+ if (work_done < budget) {
+ napi_complete(napi);
+ niu_ldg_rearm(np, lp, 1);
+ }
+ return work_done;
+}
+
+static void niu_log_rxchan_errors(struct niu *np, struct rx_ring_info *rp,
+ u64 stat)
+{
+ dev_err(np->device, PFX "%s: RX channel %u errors ( ",
+ np->dev->name, rp->rx_channel);
+
+ if (stat & RX_DMA_CTL_STAT_RBR_TMOUT)
+ printk("RBR_TMOUT ");
+ if (stat & RX_DMA_CTL_STAT_RSP_CNT_ERR)
+ printk("RSP_CNT ");
+ if (stat & RX_DMA_CTL_STAT_BYTE_EN_BUS)
+ printk("BYTE_EN_BUS ");
+ if (stat & RX_DMA_CTL_STAT_RSP_DAT_ERR)
+ printk("RSP_DAT ");
+ if (stat & RX_DMA_CTL_STAT_RCR_ACK_ERR)
+ printk("RCR_ACK ");
+ if (stat & RX_DMA_CTL_STAT_RCR_SHA_PAR)
+ printk("RCR_SHA_PAR ");
+ if (stat & RX_DMA_CTL_STAT_RBR_PRE_PAR)
+ printk("RBR_PRE_PAR ");
+ if (stat & RX_DMA_CTL_STAT_CONFIG_ERR)
+ printk("CONFIG ");
+ if (stat & RX_DMA_CTL_STAT_RCRINCON)
+ printk("RCRINCON ");
+ if (stat & RX_DMA_CTL_STAT_RCRFULL)
+ printk("RCRFULL ");
+ if (stat & RX_DMA_CTL_STAT_RBRFULL)
+ printk("RBRFULL ");
+ if (stat & RX_DMA_CTL_STAT_RBRLOGPAGE)
+ printk("RBRLOGPAGE ");
+ if (stat & RX_DMA_CTL_STAT_CFIGLOGPAGE)
+ printk("CFIGLOGPAGE ");
+ if (stat & RX_DMA_CTL_STAT_DC_FIFO_ERR)
+ printk("DC_FIDO ");
+
+ printk(")\n");
+}
+
+static int niu_rx_error(struct niu *np, struct rx_ring_info *rp)
+{
+ u64 stat = nr64(RX_DMA_CTL_STAT(rp->rx_channel));
+ int err = 0;
+
+
+ if (stat & (RX_DMA_CTL_STAT_CHAN_FATAL |
+ RX_DMA_CTL_STAT_PORT_FATAL))
+ err = -EINVAL;
+
+ if (err) {
+ dev_err(np->device, PFX "%s: RX channel %u error, stat[%llx]\n",
+ np->dev->name, rp->rx_channel,
+ (unsigned long long) stat);
+
+ niu_log_rxchan_errors(np, rp, stat);
+ }
+
+ nw64(RX_DMA_CTL_STAT(rp->rx_channel),
+ stat & RX_DMA_CTL_WRITE_CLEAR_ERRS);
+
+ return err;
+}
+
+static void niu_log_txchan_errors(struct niu *np, struct tx_ring_info *rp,
+ u64 cs)
+{
+ dev_err(np->device, PFX "%s: TX channel %u errors ( ",
+ np->dev->name, rp->tx_channel);
+
+ if (cs & TX_CS_MBOX_ERR)
+ printk("MBOX ");
+ if (cs & TX_CS_PKT_SIZE_ERR)
+ printk("PKT_SIZE ");
+ if (cs & TX_CS_TX_RING_OFLOW)
+ printk("TX_RING_OFLOW ");
+ if (cs & TX_CS_PREF_BUF_PAR_ERR)
+ printk("PREF_BUF_PAR ");
+ if (cs & TX_CS_NACK_PREF)
+ printk("NACK_PREF ");
+ if (cs & TX_CS_NACK_PKT_RD)
+ printk("NACK_PKT_RD ");
+ if (cs & TX_CS_CONF_PART_ERR)
+ printk("CONF_PART ");
+ if (cs & TX_CS_PKT_PRT_ERR)
+ printk("PKT_PTR ");
+
+ printk(")\n");
+}
+
+static int niu_tx_error(struct niu *np, struct tx_ring_info *rp)
+{
+ u64 cs, logh, logl;
+
+ cs = nr64(TX_CS(rp->tx_channel));
+ logh = nr64(TX_RNG_ERR_LOGH(rp->tx_channel));
+ logl = nr64(TX_RNG_ERR_LOGL(rp->tx_channel));
+
+ dev_err(np->device, PFX "%s: TX channel %u error, "
+ "cs[%llx] logh[%llx] logl[%llx]\n",
+ np->dev->name, rp->tx_channel,
+ (unsigned long long) cs,
+ (unsigned long long) logh,
+ (unsigned long long) logl);
+
+ niu_log_txchan_errors(np, rp, cs);
+
+ return -ENODEV;
+}
+
+static int niu_mif_interrupt(struct niu *np)
+{
+ u64 mif_status = nr64(MIF_STATUS);
+ int phy_mdint = 0;
+
+ if (np->flags & NIU_FLAGS_XMAC) {
+ u64 xrxmac_stat = nr64_mac(XRXMAC_STATUS);
+
+ if (xrxmac_stat & XRXMAC_STATUS_PHY_MDINT)
+ phy_mdint = 1;
+ }
+
+ dev_err(np->device, PFX "%s: MIF interrupt, "
+ "stat[%llx] phy_mdint(%d)\n",
+ np->dev->name, (unsigned long long) mif_status, phy_mdint);
+
+ return -ENODEV;
+}
+
+static void niu_xmac_interrupt(struct niu *np)
+{
+ struct niu_xmac_stats *mp = &np->mac_stats.xmac;
+ u64 val;
+
+ val = nr64_mac(XTXMAC_STATUS);
+ if (val & XTXMAC_STATUS_FRAME_CNT_EXP)
+ mp->tx_frames += TXMAC_FRM_CNT_COUNT;
+ if (val & XTXMAC_STATUS_BYTE_CNT_EXP)
+ mp->tx_bytes += TXMAC_BYTE_CNT_COUNT;
+ if (val & XTXMAC_STATUS_TXFIFO_XFR_ERR)
+ mp->tx_fifo_errors++;
+ if (val & XTXMAC_STATUS_TXMAC_OFLOW)
+ mp->tx_overflow_errors++;
+ if (val & XTXMAC_STATUS_MAX_PSIZE_ERR)
+ mp->tx_max_pkt_size_errors++;
+ if (val & XTXMAC_STATUS_TXMAC_UFLOW)
+ mp->tx_underflow_errors++;
+
+ val = nr64_mac(XRXMAC_STATUS);
+ if (val & XRXMAC_STATUS_LCL_FLT_STATUS)
+ mp->rx_local_faults++;
+ if (val & XRXMAC_STATUS_RFLT_DET)
+ mp->rx_remote_faults++;
+ if (val & XRXMAC_STATUS_LFLT_CNT_EXP)
+ mp->rx_link_faults += LINK_FAULT_CNT_COUNT;
+ if (val & XRXMAC_STATUS_ALIGNERR_CNT_EXP)
+ mp->rx_align_errors += RXMAC_ALIGN_ERR_CNT_COUNT;
+ if (val & XRXMAC_STATUS_RXFRAG_CNT_EXP)
+ mp->rx_frags += RXMAC_FRAG_CNT_COUNT;
+ if (val & XRXMAC_STATUS_RXMULTF_CNT_EXP)
+ mp->rx_mcasts += RXMAC_MC_FRM_CNT_COUNT;
+ if (val & XRXMAC_STATUS_RXBCAST_CNT_EXP)
+ mp->rx_bcasts += RXMAC_BC_FRM_CNT_COUNT;
+ if (val & XRXMAC_STATUS_RXBCAST_CNT_EXP)
+ mp->rx_bcasts += RXMAC_BC_FRM_CNT_COUNT;
+ if (val & XRXMAC_STATUS_RXHIST1_CNT_EXP)
+ mp->rx_hist_cnt1 += RXMAC_HIST_CNT1_COUNT;
+ if (val & XRXMAC_STATUS_RXHIST2_CNT_EXP)
+ mp->rx_hist_cnt2 += RXMAC_HIST_CNT2_COUNT;
+ if (val & XRXMAC_STATUS_RXHIST3_CNT_EXP)
+ mp->rx_hist_cnt3 += RXMAC_HIST_CNT3_COUNT;
+ if (val & XRXMAC_STATUS_RXHIST4_CNT_EXP)
+ mp->rx_hist_cnt4 += RXMAC_HIST_CNT4_COUNT;
+ if (val & XRXMAC_STATUS_RXHIST5_CNT_EXP)
+ mp->rx_hist_cnt5 += RXMAC_HIST_CNT5_COUNT;
+ if (val & XRXMAC_STATUS_RXHIST6_CNT_EXP)
+ mp->rx_hist_cnt6 += RXMAC_HIST_CNT6_COUNT;
+ if (val & XRXMAC_STATUS_RXHIST7_CNT_EXP)
+ mp->rx_hist_cnt7 += RXMAC_HIST_CNT7_COUNT;
+ if (val & XRXMAC_STAT_MSK_RXOCTET_CNT_EXP)
+ mp->rx_octets += RXMAC_BT_CNT_COUNT;
+ if (val & XRXMAC_STATUS_CVIOLERR_CNT_EXP)
+ mp->rx_code_violations += RXMAC_CD_VIO_CNT_COUNT;
+ if (val & XRXMAC_STATUS_LENERR_CNT_EXP)
+ mp->rx_len_errors += RXMAC_MPSZER_CNT_COUNT;
+ if (val & XRXMAC_STATUS_CRCERR_CNT_EXP)
+ mp->rx_crc_errors += RXMAC_CRC_ER_CNT_COUNT;
+ if (val & XRXMAC_STATUS_RXUFLOW)
+ mp->rx_underflows++;
+ if (val & XRXMAC_STATUS_RXOFLOW)
+ mp->rx_overflows++;
+
+ val = nr64_mac(XMAC_FC_STAT);
+ if (val & XMAC_FC_STAT_TX_MAC_NPAUSE)
+ mp->pause_off_state++;
+ if (val & XMAC_FC_STAT_TX_MAC_PAUSE)
+ mp->pause_on_state++;
+ if (val & XMAC_FC_STAT_RX_MAC_RPAUSE)
+ mp->pause_received++;
+}
+
+static void niu_bmac_interrupt(struct niu *np)
+{
+ struct niu_bmac_stats *mp = &np->mac_stats.bmac;
+ u64 val;
+
+ val = nr64_mac(BTXMAC_STATUS);
+ if (val & BTXMAC_STATUS_UNDERRUN)
+ mp->tx_underflow_errors++;
+ if (val & BTXMAC_STATUS_MAX_PKT_ERR)
+ mp->tx_max_pkt_size_errors++;
+ if (val & BTXMAC_STATUS_BYTE_CNT_EXP)
+ mp->tx_bytes += BTXMAC_BYTE_CNT_COUNT;
+ if (val & BTXMAC_STATUS_FRAME_CNT_EXP)
+ mp->tx_frames += BTXMAC_FRM_CNT_COUNT;
+
+ val = nr64_mac(BRXMAC_STATUS);
+ if (val & BRXMAC_STATUS_OVERFLOW)
+ mp->rx_overflows++;
+ if (val & BRXMAC_STATUS_FRAME_CNT_EXP)
+ mp->rx_frames += BRXMAC_FRAME_CNT_COUNT;
+ if (val & BRXMAC_STATUS_ALIGN_ERR_EXP)
+ mp->rx_align_errors += BRXMAC_ALIGN_ERR_CNT_COUNT;
+ if (val & BRXMAC_STATUS_CRC_ERR_EXP)
+ mp->rx_crc_errors += BRXMAC_ALIGN_ERR_CNT_COUNT;
+ if (val & BRXMAC_STATUS_LEN_ERR_EXP)
+ mp->rx_len_errors += BRXMAC_CODE_VIOL_ERR_CNT_COUNT;
+
+ val = nr64_mac(BMAC_CTRL_STATUS);
+ if (val & BMAC_CTRL_STATUS_NOPAUSE)
+ mp->pause_off_state++;
+ if (val & BMAC_CTRL_STATUS_PAUSE)
+ mp->pause_on_state++;
+ if (val & BMAC_CTRL_STATUS_PAUSE_RECV)
+ mp->pause_received++;
+}
+
+static int niu_mac_interrupt(struct niu *np)
+{
+ if (np->flags & NIU_FLAGS_XMAC)
+ niu_xmac_interrupt(np);
+ else
+ niu_bmac_interrupt(np);
+
+ return 0;
+}
+
+static void niu_log_device_error(struct niu *np, u64 stat)
+{
+ dev_err(np->device, PFX "%s: Core device errors ( ",
+ np->dev->name);
+
+ if (stat & SYS_ERR_MASK_META2)
+ printk("META2 ");
+ if (stat & SYS_ERR_MASK_META1)
+ printk("META1 ");
+ if (stat & SYS_ERR_MASK_PEU)
+ printk("PEU ");
+ if (stat & SYS_ERR_MASK_TXC)
+ printk("TXC ");
+ if (stat & SYS_ERR_MASK_RDMC)
+ printk("RDMC ");
+ if (stat & SYS_ERR_MASK_TDMC)
+ printk("TDMC ");
+ if (stat & SYS_ERR_MASK_ZCP)
+ printk("ZCP ");
+ if (stat & SYS_ERR_MASK_FFLP)
+ printk("FFLP ");
+ if (stat & SYS_ERR_MASK_IPP)
+ printk("IPP ");
+ if (stat & SYS_ERR_MASK_MAC)
+ printk("MAC ");
+ if (stat & SYS_ERR_MASK_SMX)
+ printk("SMX ");
+
+ printk(")\n");
+}
+
+static int niu_device_error(struct niu *np)
+{
+ u64 stat = nr64(SYS_ERR_STAT);
+
+ dev_err(np->device, PFX "%s: Core device error, stat[%llx]\n",
+ np->dev->name, (unsigned long long) stat);
+
+ niu_log_device_error(np, stat);
+
+ return -ENODEV;
+}
+
+static int niu_slowpath_interrupt(struct niu *np, struct niu_ldg *lp,
+ u64 v0, u64 v1, u64 v2)
+{
+
+ int i, err = 0;
+
+ lp->v0 = v0;
+ lp->v1 = v1;
+ lp->v2 = v2;
+
+ if (v1 & 0x00000000ffffffffULL) {
+ u32 rx_vec = (v1 & 0xffffffff);
+
+ for (i = 0; i < np->num_rx_rings; i++) {
+ struct rx_ring_info *rp = &np->rx_rings[i];
+
+ if (rx_vec & (1 << rp->rx_channel)) {
+ int r = niu_rx_error(np, rp);
+ if (r) {
+ err = r;
+ } else {
+ if (!v0)
+ nw64(RX_DMA_CTL_STAT(rp->rx_channel),
+ RX_DMA_CTL_STAT_MEX);
+ }
+ }
+ }
+ }
+ if (v1 & 0x7fffffff00000000ULL) {
+ u32 tx_vec = (v1 >> 32) & 0x7fffffff;
+
+ for (i = 0; i < np->num_tx_rings; i++) {
+ struct tx_ring_info *rp = &np->tx_rings[i];
+
+ if (tx_vec & (1 << rp->tx_channel)) {
+ int r = niu_tx_error(np, rp);
+ if (r)
+ err = r;
+ }
+ }
+ }
+ if ((v0 | v1) & 0x8000000000000000ULL) {
+ int r = niu_mif_interrupt(np);
+ if (r)
+ err = r;
+ }
+ if (v2) {
+ if (v2 & 0x01ef) {
+ int r = niu_mac_interrupt(np);
+ if (r)
+ err = r;
+ }
+ if (v2 & 0x0210) {
+ int r = niu_device_error(np);
+ if (r)
+ err = r;
+ }
+ }
+
+ if (err)
+ niu_enable_interrupts(np, 0);
+
+ return err;
+}
+
+static void niu_rxchan_intr(struct niu *np, struct rx_ring_info *rp,
+ int ldn)
+{
+ struct rxdma_mailbox *mbox = rp->mbox;
+ u64 stat_write, stat = le64_to_cpup(&mbox->rx_dma_ctl_stat);
+
+ stat_write = (RX_DMA_CTL_STAT_RCRTHRES |
+ RX_DMA_CTL_STAT_RCRTO);
+ nw64(RX_DMA_CTL_STAT(rp->rx_channel), stat_write);
+
+ niudbg(INTR, "%s: rxchan_intr stat[%llx]\n",
+ np->dev->name, (unsigned long long) stat);
+}
+
+static void niu_txchan_intr(struct niu *np, struct tx_ring_info *rp,
+ int ldn)
+{
+ rp->tx_cs = nr64(TX_CS(rp->tx_channel));
+
+ niudbg(INTR, "%s: txchan_intr cs[%llx]\n",
+ np->dev->name, (unsigned long long) rp->tx_cs);
+}
+
+static void __niu_fastpath_interrupt(struct niu *np, int ldg, u64 v0)
+{
+ struct niu_parent *parent = np->parent;
+ u32 rx_vec, tx_vec;
+ int i;
+
+ tx_vec = (v0 >> 32);
+ rx_vec = (v0 & 0xffffffff);
+
+ for (i = 0; i < np->num_rx_rings; i++) {
+ struct rx_ring_info *rp = &np->rx_rings[i];
+ int ldn = LDN_RXDMA(rp->rx_channel);
+
+ if (parent->ldg_map[ldn] != ldg)
+ continue;
+
+ nw64(LD_IM0(ldn), LD_IM0_MASK);
+ if (rx_vec & (1 << rp->rx_channel))
+ niu_rxchan_intr(np, rp, ldn);
+ }
+
+ for (i = 0; i < np->num_tx_rings; i++) {
+ struct tx_ring_info *rp = &np->tx_rings[i];
+ int ldn = LDN_TXDMA(rp->tx_channel);
+
+ if (parent->ldg_map[ldn] != ldg)
+ continue;
+
+ nw64(LD_IM0(ldn), LD_IM0_MASK);
+ if (tx_vec & (1 << rp->tx_channel))
+ niu_txchan_intr(np, rp, ldn);
+ }
+}
+
+static void niu_schedule_napi(struct niu *np, struct niu_ldg *lp,
+ u64 v0, u64 v1, u64 v2)
+{
+ if (likely(napi_schedule_prep(&lp->napi))) {
+ lp->v0 = v0;
+ lp->v1 = v1;
+ lp->v2 = v2;
+ __niu_fastpath_interrupt(np, lp->ldg_num, v0);
+ __napi_schedule(&lp->napi);
+ }
+}
+
+static irqreturn_t niu_interrupt(int irq, void *dev_id)
+{
+ struct niu_ldg *lp = dev_id;
+ struct niu *np = lp->np;
+ int ldg = lp->ldg_num;
+ unsigned long flags;
+ u64 v0, v1, v2;
+
+ if (netif_msg_intr(np))
+ printk(KERN_DEBUG PFX "niu_interrupt() ldg[%p](%d) ",
+ lp, ldg);
+
+ spin_lock_irqsave(&np->lock, flags);
+
+ v0 = nr64(LDSV0(ldg));
+ v1 = nr64(LDSV1(ldg));
+ v2 = nr64(LDSV2(ldg));
+
+ if (netif_msg_intr(np))
+ printk("v0[%llx] v1[%llx] v2[%llx]\n",
+ (unsigned long long) v0,
+ (unsigned long long) v1,
+ (unsigned long long) v2);
+
+ if (unlikely(!v0 && !v1 && !v2)) {
+ spin_unlock_irqrestore(&np->lock, flags);
+ return IRQ_NONE;
+ }
+
+ if (unlikely((v0 & ((u64)1 << LDN_MIF)) || v1 || v2)) {
+ int err = niu_slowpath_interrupt(np, lp, v0, v1, v2);
+ if (err)
+ goto out;
+ }
+ if (likely(v0 & ~((u64)1 << LDN_MIF)))
+ niu_schedule_napi(np, lp, v0, v1, v2);
+ else
+ niu_ldg_rearm(np, lp, 1);
+out:
+ spin_unlock_irqrestore(&np->lock, flags);
+
+ return IRQ_HANDLED;
+}
+
+static void niu_free_rx_ring_info(struct niu *np, struct rx_ring_info *rp)
+{
+ if (rp->mbox) {
+ np->ops->free_coherent(np->device,
+ sizeof(struct rxdma_mailbox),
+ rp->mbox, rp->mbox_dma);
+ rp->mbox = NULL;
+ }
+ if (rp->rcr) {
+ np->ops->free_coherent(np->device,
+ MAX_RCR_RING_SIZE * sizeof(__le64),
+ rp->rcr, rp->rcr_dma);
+ rp->rcr = NULL;
+ rp->rcr_table_size = 0;
+ rp->rcr_index = 0;
+ }
+ if (rp->rbr) {
+ niu_rbr_free(np, rp);
+
+ np->ops->free_coherent(np->device,
+ MAX_RBR_RING_SIZE * sizeof(__le32),
+ rp->rbr, rp->rbr_dma);
+ rp->rbr = NULL;
+ rp->rbr_table_size = 0;
+ rp->rbr_index = 0;
+ }
+ kfree(rp->rxhash);
+ rp->rxhash = NULL;
+}
+
+static void niu_free_tx_ring_info(struct niu *np, struct tx_ring_info *rp)
+{
+ if (rp->mbox) {
+ np->ops->free_coherent(np->device,
+ sizeof(struct txdma_mailbox),
+ rp->mbox, rp->mbox_dma);
+ rp->mbox = NULL;
+ }
+ if (rp->descr) {
+ int i;
+
+ for (i = 0; i < MAX_TX_RING_SIZE; i++) {
+ if (rp->tx_buffs[i].skb)
+ (void) release_tx_packet(np, rp, i);
+ }
+
+ np->ops->free_coherent(np->device,
+ MAX_TX_RING_SIZE * sizeof(__le64),
+ rp->descr, rp->descr_dma);
+ rp->descr = NULL;
+ rp->pending = 0;
+ rp->prod = 0;
+ rp->cons = 0;
+ rp->wrap_bit = 0;
+ }
+}
+
+static void niu_free_channels(struct niu *np)
+{
+ int i;
+
+ if (np->rx_rings) {
+ for (i = 0; i < np->num_rx_rings; i++) {
+ struct rx_ring_info *rp = &np->rx_rings[i];
+
+ niu_free_rx_ring_info(np, rp);
+ }
+ kfree(np->rx_rings);
+ np->rx_rings = NULL;
+ np->num_rx_rings = 0;
+ }
+
+ if (np->tx_rings) {
+ for (i = 0; i < np->num_tx_rings; i++) {
+ struct tx_ring_info *rp = &np->tx_rings[i];
+
+ niu_free_tx_ring_info(np, rp);
+ }
+ kfree(np->tx_rings);
+ np->tx_rings = NULL;
+ np->num_tx_rings = 0;
+ }
+}
+
+static int niu_alloc_rx_ring_info(struct niu *np,
+ struct rx_ring_info *rp)
+{
+ BUILD_BUG_ON(sizeof(struct rxdma_mailbox) != 64);
+
+ rp->rxhash = kzalloc(MAX_RBR_RING_SIZE * sizeof(struct page *),
+ GFP_KERNEL);
+ if (!rp->rxhash)
+ return -ENOMEM;
+
+ rp->mbox = np->ops->alloc_coherent(np->device,
+ sizeof(struct rxdma_mailbox),
+ &rp->mbox_dma, GFP_KERNEL);
+ if (!rp->mbox)
+ return -ENOMEM;
+ if ((unsigned long)rp->mbox & (64UL - 1)) {
+ dev_err(np->device, PFX "%s: Coherent alloc gives misaligned "
+ "RXDMA mailbox %p\n", np->dev->name, rp->mbox);
+ return -EINVAL;
+ }
+
+ rp->rcr = np->ops->alloc_coherent(np->device,
+ MAX_RCR_RING_SIZE * sizeof(__le64),
+ &rp->rcr_dma, GFP_KERNEL);
+ if (!rp->rcr)
+ return -ENOMEM;
+ if ((unsigned long)rp->rcr & (64UL - 1)) {
+ dev_err(np->device, PFX "%s: Coherent alloc gives misaligned "
+ "RXDMA RCR table %p\n", np->dev->name, rp->rcr);
+ return -EINVAL;
+ }
+ rp->rcr_table_size = MAX_RCR_RING_SIZE;
+ rp->rcr_index = 0;
+
+ rp->rbr = np->ops->alloc_coherent(np->device,
+ MAX_RBR_RING_SIZE * sizeof(__le32),