SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/staging/octeon/ethernet-rx.c

   1 /**********************************************************************
   2  * Author: Cavium Networks
   3  *
   4  * Contact: support@caviumnetworks.com
   5  * This file is part of the OCTEON SDK
   6  *
   7  * Copyright (c) 2003-2007 Cavium Networks
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more
  17  * details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this file; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  22  * or visit http://www.gnu.org/licenses/.
  23  *
  24  * This file may also be available under a different license from Cavium.
  25  * Contact Cavium Networks for more information
  26 **********************************************************************/
  27 #include <linux/module.h>
  28 #include <linux/kernel.h>
  29 #include <linux/cache.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/init.h>
  32 #include <linux/etherdevice.h>
  33 #include <linux/ip.h>
  34 #include <linux/string.h>
  35 #include <linux/prefetch.h>
  36 #include <linux/ethtool.h>
  37 #include <linux/mii.h>
  38 #include <linux/seq_file.h>
  39 #include <linux/proc_fs.h>
  40 #include <net/dst.h>
  41 #ifdef CONFIG_XFRM
  42 #include <linux/xfrm.h>
  43 #include <net/xfrm.h>
  44 #endif /* CONFIG_XFRM */
  45
  46 #include <asm/atomic.h>
  47
  48 #include <asm/octeon/octeon.h>
  49
  50 #include "ethernet-defines.h"
  51 #include "octeon-ethernet.h"
  52 #include "ethernet-mem.h"
  53 #include "ethernet-util.h"
  54
  55 #include "cvmx-helper.h"
  56 #include "cvmx-wqe.h"
  57 #include "cvmx-fau.h"
  58 #include "cvmx-pow.h"
  59 #include "cvmx-pip.h"
  60 #include "cvmx-scratch.h"
  61
  62 #include "cvmx-gmxx-defs.h"
  63
  64 struct cvm_tasklet_wrapper {
  65         struct tasklet_struct t;
  66 };
  67
  68 /*
  69  * Aligning the tasklet_struct on cachline boundries seems to decrease
  70  * throughput even though in theory it would reduce contantion on the
  71  * cache lines containing the locks.
  72  */
  73
  74 static struct cvm_tasklet_wrapper cvm_oct_tasklet[NR_CPUS];
  75
  76 /**
  77  * Interrupt handler. The interrupt occurs whenever the POW
  78  * transitions from 0->1 packets in our group.
  79  *
  80  * @cpl:
  81  * @dev_id:
  82  * @regs:
  83  * Returns
  84  */
  85 irqreturn_t cvm_oct_do_interrupt(int cpl, void *dev_id)
  86 {
  87         /* Acknowledge the interrupt */
  88         if (INTERRUPT_LIMIT)
  89                 cvmx_write_csr(CVMX_POW_WQ_INT, 1 << pow_receive_group);
  90         else
  91                 cvmx_write_csr(CVMX_POW_WQ_INT, 0x10001 << pow_receive_group);
  92         preempt_disable();
  93         tasklet_schedule(&cvm_oct_tasklet[smp_processor_id()].t);
  94         preempt_enable();
  95         return IRQ_HANDLED;
  96 }
  97
  98 #ifdef CONFIG_NET_POLL_CONTROLLER
  99 /**
 100  * This is called when the kernel needs to manually poll the
 101  * device. For Octeon, this is simply calling the interrupt
 102  * handler. We actually poll all the devices, not just the
 103  * one supplied.
 104  *
 105  * @dev:    Device to poll. Unused
 106  */
 107 void cvm_oct_poll_controller(struct net_device *dev)
 108 {
 109         preempt_disable();
 110         tasklet_schedule(&cvm_oct_tasklet[smp_processor_id()].t);
 111         preempt_enable();
 112 }
 113 #endif
 114
 115 /**
 116  * This is called on receive errors, and determines if the packet
 117  * can be dropped early-on in cvm_oct_tasklet_rx().
 118  *
 119  * @work: Work queue entry pointing to the packet.
 120  * Returns Non-zero if the packet can be dropped, zero otherwise.
 121  */
 122 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work)
 123 {
 124         if ((work->word2.snoip.err_code == 10) && (work->len <= 64)) {
 125                 /*
 126                  * Ignore length errors on min size packets. Some
 127                  * equipment incorrectly pads packets to 64+4FCS
 128                  * instead of 60+4FCS.  Note these packets still get
 129                  * counted as frame errors.
 130                  */
 131         } else
 132             if (USE_10MBPS_PREAMBLE_WORKAROUND
 133                 && ((work->word2.snoip.err_code == 5)
 134                     || (work->word2.snoip.err_code == 7))) {
 135
 136                 /*
 137                  * We received a packet with either an alignment error
 138                  * or a FCS error. This may be signalling that we are
 139                  * running 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK}
 140                  * off. If this is the case we need to parse the
 141                  * packet to determine if we can remove a non spec
 142                  * preamble and generate a correct packet.
 143                  */
 144                 int interface = cvmx_helper_get_interface_num(work->ipprt);
 145                 int index = cvmx_helper_get_interface_index_num(work->ipprt);
 146                 union cvmx_gmxx_rxx_frm_ctl gmxx_rxx_frm_ctl;
 147                 gmxx_rxx_frm_ctl.u64 =
 148                     cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface));
 149                 if (gmxx_rxx_frm_ctl.s.pre_chk == 0) {
 150
 151                         uint8_t *ptr =
 152                             cvmx_phys_to_ptr(work->packet_ptr.s.addr);
 153                         int i = 0;
 154
 155                         while (i < work->len - 1) {
 156                                 if (*ptr != 0x55)
 157                                         break;
 158                                 ptr++;
 159                                 i++;
 160                         }
 161
 162                         if (*ptr == 0xd5) {
 163                                 /*
 164                                    DEBUGPRINT("Port %d received 0xd5 preamble\n", work->ipprt);
 165                                  */
 166                                 work->packet_ptr.s.addr += i + 1;
 167                                 work->len -= i + 5;
 168                         } else if ((*ptr & 0xf) == 0xd) {
 169                                 /*
 170                                    DEBUGPRINT("Port %d received 0x?d preamble\n", work->ipprt);
 171                                  */
 172                                 work->packet_ptr.s.addr += i;
 173                                 work->len -= i + 4;
 174                                 for (i = 0; i < work->len; i++) {
 175                                         *ptr =
 176                                             ((*ptr & 0xf0) >> 4) |
 177                                             ((*(ptr + 1) & 0xf) << 4);
 178                                         ptr++;
 179                                 }
 180                         } else {
 181                                 DEBUGPRINT("Port %d unknown preamble, packet "
 182                                            "dropped\n",
 183                                      work->ipprt);
 184                                 /*
 185                                    cvmx_helper_dump_packet(work);
 186                                  */
 187                                 cvm_oct_free_work(work);
 188                                 return 1;
 189                         }
 190                 }
 191         } else {
 192                 DEBUGPRINT("Port %d receive error code %d, packet dropped\n",
 193                            work->ipprt, work->word2.snoip.err_code);
 194                 cvm_oct_free_work(work);
 195                 return 1;
 196         }
 197
 198         return 0;
 199 }
 200
 201 /**
 202  * Tasklet function that is scheduled on a core when an interrupt occurs.
 203  *
 204  * @unused:
 205  */
 206 void cvm_oct_tasklet_rx(unsigned long unused)
 207 {
 208         const int coreid = cvmx_get_core_num();
 209         uint64_t old_group_mask;
 210         uint64_t old_scratch;
 211         int rx_count = 0;
 212         int number_to_free;
 213         int num_freed;
 214         int packet_not_copied;
 215
 216         /* Prefetch cvm_oct_device since we know we need it soon */
 217         prefetch(cvm_oct_device);
 218
 219         if (USE_ASYNC_IOBDMA) {
 220                 /* Save scratch in case userspace is using it */
 221                 CVMX_SYNCIOBDMA;
 222                 old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH);
 223         }
 224
 225         /* Only allow work for our group (and preserve priorities) */
 226         old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid));
 227         cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid),
 228                        (old_group_mask & ~0xFFFFull) | 1 << pow_receive_group);
 229
 230         if (USE_ASYNC_IOBDMA)
 231                 cvmx_pow_work_request_async(CVMX_SCR_SCRATCH, CVMX_POW_NO_WAIT);
 232
 233         while (1) {
 234                 struct sk_buff *skb = NULL;
 235                 int skb_in_hw;
 236                 cvmx_wqe_t *work;
 237
 238                 if (USE_ASYNC_IOBDMA) {
 239                         work = cvmx_pow_work_response_async(CVMX_SCR_SCRATCH);
 240                 } else {
 241                         if ((INTERRUPT_LIMIT == 0)
 242                             || likely(rx_count < MAX_RX_PACKETS))
 243                                 work =
 244                                     cvmx_pow_work_request_sync
 245                                     (CVMX_POW_NO_WAIT);
 246                         else
 247                                 work = NULL;
 248                 }
 249                 prefetch(work);
 250                 if (work == NULL)
 251                         break;
 252
 253                 /*
 254                  * Limit each core to processing MAX_RX_PACKETS
 255                  * packets without a break.  This way the RX can't
 256                  * starve the TX task.
 257                  */
 258                 if (USE_ASYNC_IOBDMA) {
 259
 260                         if ((INTERRUPT_LIMIT == 0)
 261                             || likely(rx_count < MAX_RX_PACKETS))
 262                                 cvmx_pow_work_request_async_nocheck
 263                                     (CVMX_SCR_SCRATCH, CVMX_POW_NO_WAIT);
 264                         else {
 265                                 cvmx_scratch_write64(CVMX_SCR_SCRATCH,
 266                                                      0x8000000000000000ull);
 267                                 cvmx_pow_tag_sw_null_nocheck();
 268                         }
 269                 }
 270
 271                 skb_in_hw = USE_SKBUFFS_IN_HW && work->word2.s.bufs == 1;
 272                 if (likely(skb_in_hw)) {
 273                         skb =
 274                             *(struct sk_buff
 275                               **)(cvm_oct_get_buffer_ptr(work->packet_ptr) -
 276                                   sizeof(void *));
 277                         prefetch(&skb->head);
 278                         prefetch(&skb->len);
 279                 }
 280                 prefetch(cvm_oct_device[work->ipprt]);
 281
 282                 rx_count++;
 283                 /* Immediately throw away all packets with receive errors */
 284                 if (unlikely(work->word2.snoip.rcv_error)) {
 285                         if (cvm_oct_check_rcv_error(work))
 286                                 continue;
 287                 }
 288
 289                 /*
 290                  * We can only use the zero copy path if skbuffs are
 291                  * in the FPA pool and the packet fits in a single
 292                  * buffer.
 293                  */
 294                 if (likely(skb_in_hw)) {
 295                         /*
 296                          * This calculation was changed in case the
 297                          * skb header is using a different address
 298                          * aliasing type than the buffer. It doesn't
 299                          * make any differnece now, but the new one is
 300                          * more correct.
 301                          */
 302                         skb->data =
 303                             skb->head + work->packet_ptr.s.addr -
 304                             cvmx_ptr_to_phys(skb->head);
 305                         prefetch(skb->data);
 306                         skb->len = work->len;
 307                         skb_set_tail_pointer(skb, skb->len);
 308                         packet_not_copied = 1;
 309                 } else {
 310
 311                         /*
 312                          * We have to copy the packet. First allocate
 313                          * an skbuff for it.
 314                          */
 315                         skb = dev_alloc_skb(work->len);
 316                         if (!skb) {
 317                                 DEBUGPRINT("Port %d failed to allocate "
 318                                            "skbuff, packet dropped\n",
 319                                      work->ipprt);
 320                                 cvm_oct_free_work(work);
 321                                 continue;
 322                         }
 323
 324                         /*
 325                          * Check if we've received a packet that was
 326                          * entirely stored in the work entry. This is
 327                          * untested.
 328                          */
 329                         if (unlikely(work->word2.s.bufs == 0)) {
 330                                 uint8_t *ptr = work->packet_data;
 331
 332                                 if (likely(!work->word2.s.not_IP)) {
 333                                         /*
 334                                          * The beginning of the packet
 335                                          * moves for IP packets.
 336                                          */
 337                                         if (work->word2.s.is_v6)
 338                                                 ptr += 2;
 339                                         else
 340                                                 ptr += 6;
 341                                 }
 342                                 memcpy(skb_put(skb, work->len), ptr, work->len);
 343                                 /* No packet buffers to free */
 344                         } else {
 345                                 int segments = work->word2.s.bufs;
 346                                 union cvmx_buf_ptr segment_ptr =
 347                                         work->packet_ptr;
 348                                 int len = work->len;
 349
 350                                 while (segments--) {
 351                                         union cvmx_buf_ptr next_ptr =
 352                                             *(union cvmx_buf_ptr *)
 353                                             cvmx_phys_to_ptr(segment_ptr.s.
 354                                                              addr - 8);
 355                         /*
 356                          * Octeon Errata PKI-100: The segment size is
 357                          * wrong. Until it is fixed, calculate the
 358                          * segment size based on the packet pool
 359                          * buffer size. When it is fixed, the
 360                          * following line should be replaced with this
 361                          * one: int segment_size =
 362                          * segment_ptr.s.size;
 363                          */
 364                                         int segment_size =
 365                                             CVMX_FPA_PACKET_POOL_SIZE -
 366                                             (segment_ptr.s.addr -
 367                                              (((segment_ptr.s.addr >> 7) -
 368                                                segment_ptr.s.back) << 7));
 369                                         /* Don't copy more than what is left
 370                                            in the packet */
 371                                         if (segment_size > len)
 372                                                 segment_size = len;
 373                                         /* Copy the data into the packet */
 374                                         memcpy(skb_put(skb, segment_size),
 375                                                cvmx_phys_to_ptr(segment_ptr.s.
 376                                                                 addr),
 377                                                segment_size);
 378                                         /* Reduce the amount of bytes left
 379                                            to copy */
 380                                         len -= segment_size;
 381                                         segment_ptr = next_ptr;
 382                                 }
 383                         }
 384                         packet_not_copied = 0;
 385                 }
 386
 387                 if (likely((work->ipprt < TOTAL_NUMBER_OF_PORTS) &&
 388                            cvm_oct_device[work->ipprt])) {
 389                         struct net_device *dev = cvm_oct_device[work->ipprt];
 390                         struct octeon_ethernet *priv = netdev_priv(dev);
 391
 392                         /* Only accept packets for devices
 393                            that are currently up */
 394                         if (likely(dev->flags & IFF_UP)) {
 395                                 skb->protocol = eth_type_trans(skb, dev);
 396                                 skb->dev = dev;
 397
 398                                 if (unlikely
 399                                     (work->word2.s.not_IP
 400                                      || work->word2.s.IP_exc
 401                                      || work->word2.s.L4_error))
 402                                         skb->ip_summed = CHECKSUM_NONE;
 403                                 else
 404                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 405
 406                                 /* Increment RX stats for virtual ports */
 407                                 if (work->ipprt >= CVMX_PIP_NUM_INPUT_PORTS) {
 408 #ifdef CONFIG_64BIT
 409                                         atomic64_add(1, (atomic64_t *)&priv->stats.rx_packets);
 410                                         atomic64_add(skb->len, (atomic64_t *)&priv->stats.rx_bytes);
 411 #else
 412                                         atomic_add(1, (atomic_t *)&priv->stats.rx_packets);
 413                                         atomic_add(skb->len, (atomic_t *)&priv->stats.rx_bytes);
 414 #endif
 415                                 }
 416                                 netif_receive_skb(skb);
 417                         } else {
 418                                 /*
 419                                  * Drop any packet received for a
 420                                  * device that isn't up.
 421                                  */
 422                                 /*
 423                                    DEBUGPRINT("%s: Device not up, packet dropped\n",
 424                                    dev->name);
 425                                  */
 426 #ifdef CONFIG_64BIT
 427                                 atomic64_add(1, (atomic64_t *)&priv->stats.rx_dropped);
 428 #else
 429                                 atomic_add(1, (atomic_t *)&priv->stats.rx_dropped);
 430 #endif
 431                                 dev_kfree_skb_irq(skb);
 432                         }
 433                 } else {
 434                         /*
 435                          * Drop any packet received for a device that
 436                          * doesn't exist.
 437                          */
 438                         DEBUGPRINT("Port %d not controlled by Linux, packet "
 439                                    "dropped\n",
 440                              work->ipprt);
 441                         dev_kfree_skb_irq(skb);
 442                 }
 443                 /*
 444                  * Check to see if the skbuff and work share the same
 445                  * packet buffer.
 446                  */
 447                 if (USE_SKBUFFS_IN_HW && likely(packet_not_copied)) {
 448                         /*
 449                          * This buffer needs to be replaced, increment
 450                          * the number of buffers we need to free by
 451                          * one.
 452                          */
 453                         cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
 454                                               1);
 455
 456                         cvmx_fpa_free(work, CVMX_FPA_WQE_POOL,
 457                                       DONT_WRITEBACK(1));
 458                 } else {
 459                         cvm_oct_free_work(work);
 460                 }
 461         }
 462
 463         /* Restore the original POW group mask */
 464         cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask);
 465         if (USE_ASYNC_IOBDMA) {
 466                 /* Restore the scratch area */
 467                 cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch);
 468         }
 469
 470         if (USE_SKBUFFS_IN_HW) {
 471                 /* Refill the packet buffer pool */
 472                 number_to_free =
 473                     cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
 474
 475                 if (number_to_free > 0) {
 476                         cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
 477                                               -number_to_free);
 478                         num_freed =
 479                             cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL,
 480                                                  CVMX_FPA_PACKET_POOL_SIZE,
 481                                                  number_to_free);
 482                         if (num_freed != number_to_free) {
 483                                 cvmx_fau_atomic_add32
 484                                     (FAU_NUM_PACKET_BUFFERS_TO_FREE,
 485                                      number_to_free - num_freed);
 486                         }
 487                 }
 488         }
 489 }
 490
 491 void cvm_oct_rx_initialize(void)
 492 {
 493         int i;
 494         /* Initialize all of the tasklets */
 495         for (i = 0; i < NR_CPUS; i++)
 496                 tasklet_init(&cvm_oct_tasklet[i].t, cvm_oct_tasklet_rx, 0);
 497 }
 498
 499 void cvm_oct_rx_shutdown(void)
 500 {
 501         int i;
 502         /* Shutdown all of the tasklets */
 503         for (i = 0; i < NR_CPUS; i++)
 504                 tasklet_kill(&cvm_oct_tasklet[i].t);
 505 }