SAFE public projects git trees. - safe/jmp/linux-2.6/blob - net/packet/af_packet.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              PACKET - implements raw packet sockets.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *
  12  * Fixes:
  13  *              Alan Cox        :       verify_area() now used correctly
  14  *              Alan Cox        :       new skbuff lists, look ma no backlogs!
  15  *              Alan Cox        :       tidied skbuff lists.
  16  *              Alan Cox        :       Now uses generic datagram routines I
  17  *                                      added. Also fixed the peek/read crash
  18  *                                      from all old Linux datagram code.
  19  *              Alan Cox        :       Uses the improved datagram code.
  20  *              Alan Cox        :       Added NULL's for socket options.
  21  *              Alan Cox        :       Re-commented the code.
  22  *              Alan Cox        :       Use new kernel side addressing
  23  *              Rob Janssen     :       Correct MTU usage.
  24  *              Dave Platt      :       Counter leaks caused by incorrect
  25  *                                      interrupt locking and some slightly
  26  *                                      dubious gcc output. Can you read
  27  *                                      compiler: it said _VOLATILE_
  28  *      Richard Kooijman        :       Timestamp fixes.
  29  *              Alan Cox        :       New buffers. Use sk->mac.raw.
  30  *              Alan Cox        :       sendmsg/recvmsg support.
  31  *              Alan Cox        :       Protocol setting support
  32  *      Alexey Kuznetsov        :       Untied from IPv4 stack.
  33  *      Cyrus Durgin            :       Fixed kerneld for kmod.
  34  *      Michal Ostrowski        :       Module initialization cleanup.
  35  *         Ulises Alonso        :       Frame number limit removal and
  36  *                                      packet_set_ring memory leak.
  37  *              Eric Biederman  :       Allow for > 8 byte hardware addresses.
  38  *                                      The convention is that longer addresses
  39  *                                      will simply extend the hardware address
  40  *                                      byte arrays at the end of sockaddr_ll
  41  *                                      and packet_mreq.
  42  *              Johann Baudy    :       Added TX RING.
  43  *
  44  *              This program is free software; you can redistribute it and/or
  45  *              modify it under the terms of the GNU General Public License
  46  *              as published by the Free Software Foundation; either version
  47  *              2 of the License, or (at your option) any later version.
  48  *
  49  */
  50
  51 #include <linux/types.h>
  52 #include <linux/mm.h>
  53 #include <linux/capability.h>
  54 #include <linux/fcntl.h>
  55 #include <linux/socket.h>
  56 #include <linux/in.h>
  57 #include <linux/inet.h>
  58 #include <linux/netdevice.h>
  59 #include <linux/if_packet.h>
  60 #include <linux/wireless.h>
  61 #include <linux/kernel.h>
  62 #include <linux/kmod.h>
  63 #include <net/net_namespace.h>
  64 #include <net/ip.h>
  65 #include <net/protocol.h>
  66 #include <linux/skbuff.h>
  67 #include <net/sock.h>
  68 #include <linux/errno.h>
  69 #include <linux/timer.h>
  70 #include <asm/system.h>
  71 #include <asm/uaccess.h>
  72 #include <asm/ioctls.h>
  73 #include <asm/page.h>
  74 #include <asm/cacheflush.h>
  75 #include <asm/io.h>
  76 #include <linux/proc_fs.h>
  77 #include <linux/seq_file.h>
  78 #include <linux/poll.h>
  79 #include <linux/module.h>
  80 #include <linux/init.h>
  81 #include <linux/mutex.h>
  82
  83 #ifdef CONFIG_INET
  84 #include <net/inet_common.h>
  85 #endif
  86
  87 /*
  88    Assumptions:
  89    - if device has no dev->hard_header routine, it adds and removes ll header
  90      inside itself. In this case ll header is invisible outside of device,
  91      but higher levels still should reserve dev->hard_header_len.
  92      Some devices are enough clever to reallocate skb, when header
  93      will not fit to reserved space (tunnel), another ones are silly
  94      (PPP).
  95    - packet socket receives packets with pulled ll header,
  96      so that SOCK_RAW should push it back.
  97
  98 On receive:
  99 -----------
 100
 101 Incoming, dev->hard_header!=NULL
 102    mac_header -> ll header
 103    data       -> data
 104
 105 Outgoing, dev->hard_header!=NULL
 106    mac_header -> ll header
 107    data       -> ll header
 108
 109 Incoming, dev->hard_header==NULL
 110    mac_header -> UNKNOWN position. It is very likely, that it points to ll
 111                  header.  PPP makes it, that is wrong, because introduce
 112                  assymetry between rx and tx paths.
 113    data       -> data
 114
 115 Outgoing, dev->hard_header==NULL
 116    mac_header -> data. ll header is still not built!
 117    data       -> data
 118
 119 Resume
 120   If dev->hard_header==NULL we are unlikely to restore sensible ll header.
 121
 122
 123 On transmit:
 124 ------------
 125
 126 dev->hard_header != NULL
 127    mac_header -> ll header
 128    data       -> ll header
 129
 130 dev->hard_header == NULL (ll header is added by device, we cannot control it)
 131    mac_header -> data
 132    data       -> data
 133
 134    We should set nh.raw on output to correct posistion,
 135    packet classifier depends on it.
 136  */
 137
 138 /* Private packet socket structures. */
 139
 140 struct packet_mclist {
 141         struct packet_mclist    *next;
 142         int                     ifindex;
 143         int                     count;
 144         unsigned short          type;
 145         unsigned short          alen;
 146         unsigned char           addr[MAX_ADDR_LEN];
 147 };
 148 /* identical to struct packet_mreq except it has
 149  * a longer address field.
 150  */
 151 struct packet_mreq_max {
 152         int             mr_ifindex;
 153         unsigned short  mr_type;
 154         unsigned short  mr_alen;
 155         unsigned char   mr_address[MAX_ADDR_LEN];
 156 };
 157
 158 #ifdef CONFIG_PACKET_MMAP
 159 static int packet_set_ring(struct sock *sk, struct tpacket_req *req,
 160                 int closing, int tx_ring);
 161
 162 struct packet_ring_buffer {
 163         char                    **pg_vec;
 164         unsigned int            head;
 165         unsigned int            frames_per_block;
 166         unsigned int            frame_size;
 167         unsigned int            frame_max;
 168
 169         unsigned int            pg_vec_order;
 170         unsigned int            pg_vec_pages;
 171         unsigned int            pg_vec_len;
 172
 173         atomic_t                pending;
 174 };
 175
 176 struct packet_sock;
 177 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
 178 #endif
 179
 180 static void packet_flush_mclist(struct sock *sk);
 181
 182 struct packet_sock {
 183         /* struct sock has to be the first member of packet_sock */
 184         struct sock             sk;
 185         struct tpacket_stats    stats;
 186 #ifdef CONFIG_PACKET_MMAP
 187         struct packet_ring_buffer       rx_ring;
 188         struct packet_ring_buffer       tx_ring;
 189         int                     copy_thresh;
 190 #endif
 191         struct packet_type      prot_hook;
 192         spinlock_t              bind_lock;
 193         struct mutex            pg_vec_lock;
 194         unsigned int            running:1,      /* prot_hook is attached*/
 195                                 auxdata:1,
 196                                 origdev:1;
 197         int                     ifindex;        /* bound device         */
 198         __be16                  num;
 199         struct packet_mclist    *mclist;
 200 #ifdef CONFIG_PACKET_MMAP
 201         atomic_t                mapped;
 202         enum tpacket_versions   tp_version;
 203         unsigned int            tp_hdrlen;
 204         unsigned int            tp_reserve;
 205         unsigned int            tp_loss:1;
 206 #endif
 207 };
 208
 209 struct packet_skb_cb {
 210         unsigned int origlen;
 211         union {
 212                 struct sockaddr_pkt pkt;
 213                 struct sockaddr_ll ll;
 214         } sa;
 215 };
 216
 217 #define PACKET_SKB_CB(__skb)    ((struct packet_skb_cb *)((__skb)->cb))
 218
 219 #ifdef CONFIG_PACKET_MMAP
 220
 221 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
 222 {
 223         union {
 224                 struct tpacket_hdr *h1;
 225                 struct tpacket2_hdr *h2;
 226                 void *raw;
 227         } h;
 228
 229         h.raw = frame;
 230         switch (po->tp_version) {
 231         case TPACKET_V1:
 232                 h.h1->tp_status = status;
 233                 flush_dcache_page(virt_to_page(&h.h1->tp_status));
 234                 break;
 235         case TPACKET_V2:
 236                 h.h2->tp_status = status;
 237                 flush_dcache_page(virt_to_page(&h.h2->tp_status));
 238                 break;
 239         default:
 240                 pr_err("TPACKET version not supported\n");
 241                 BUG();
 242         }
 243
 244         smp_wmb();
 245 }
 246
 247 static int __packet_get_status(struct packet_sock *po, void *frame)
 248 {
 249         union {
 250                 struct tpacket_hdr *h1;
 251                 struct tpacket2_hdr *h2;
 252                 void *raw;
 253         } h;
 254
 255         smp_rmb();
 256
 257         h.raw = frame;
 258         switch (po->tp_version) {
 259         case TPACKET_V1:
 260                 flush_dcache_page(virt_to_page(&h.h1->tp_status));
 261                 return h.h1->tp_status;
 262         case TPACKET_V2:
 263                 flush_dcache_page(virt_to_page(&h.h2->tp_status));
 264                 return h.h2->tp_status;
 265         default:
 266                 pr_err("TPACKET version not supported\n");
 267                 BUG();
 268                 return 0;
 269         }
 270 }
 271
 272 static void *packet_lookup_frame(struct packet_sock *po,
 273                 struct packet_ring_buffer *rb,
 274                 unsigned int position,
 275                 int status)
 276 {
 277         unsigned int pg_vec_pos, frame_offset;
 278         union {
 279                 struct tpacket_hdr *h1;
 280                 struct tpacket2_hdr *h2;
 281                 void *raw;
 282         } h;
 283
 284         pg_vec_pos = position / rb->frames_per_block;
 285         frame_offset = position % rb->frames_per_block;
 286
 287         h.raw = rb->pg_vec[pg_vec_pos] + (frame_offset * rb->frame_size);
 288
 289         if (status != __packet_get_status(po, h.raw))
 290                 return NULL;
 291
 292         return h.raw;
 293 }
 294
 295 static inline void *packet_current_frame(struct packet_sock *po,
 296                 struct packet_ring_buffer *rb,
 297                 int status)
 298 {
 299         return packet_lookup_frame(po, rb, rb->head, status);
 300 }
 301
 302 static inline void *packet_previous_frame(struct packet_sock *po,
 303                 struct packet_ring_buffer *rb,
 304                 int status)
 305 {
 306         unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
 307         return packet_lookup_frame(po, rb, previous, status);
 308 }
 309
 310 static inline void packet_increment_head(struct packet_ring_buffer *buff)
 311 {
 312         buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
 313 }
 314
 315 #endif
 316
 317 static inline struct packet_sock *pkt_sk(struct sock *sk)
 318 {
 319         return (struct packet_sock *)sk;
 320 }
 321
 322 static void packet_sock_destruct(struct sock *sk)
 323 {
 324         WARN_ON(atomic_read(&sk->sk_rmem_alloc));
 325         WARN_ON(atomic_read(&sk->sk_wmem_alloc));
 326
 327         if (!sock_flag(sk, SOCK_DEAD)) {
 328                 pr_err("Attempt to release alive packet socket: %p\n", sk);
 329                 return;
 330         }
 331
 332         sk_refcnt_debug_dec(sk);
 333 }
 334
 335
 336 static const struct proto_ops packet_ops;
 337
 338 static const struct proto_ops packet_ops_spkt;
 339
 340 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
 341                            struct packet_type *pt, struct net_device *orig_dev)
 342 {
 343         struct sock *sk;
 344         struct sockaddr_pkt *spkt;
 345
 346         /*
 347          *      When we registered the protocol we saved the socket in the data
 348          *      field for just this event.
 349          */
 350
 351         sk = pt->af_packet_priv;
 352
 353         /*
 354          *      Yank back the headers [hope the device set this
 355          *      right or kerboom...]
 356          *
 357          *      Incoming packets have ll header pulled,
 358          *      push it back.
 359          *
 360          *      For outgoing ones skb->data == skb_mac_header(skb)
 361          *      so that this procedure is noop.
 362          */
 363
 364         if (skb->pkt_type == PACKET_LOOPBACK)
 365                 goto out;
 366
 367         if (dev_net(dev) != sock_net(sk))
 368                 goto out;
 369
 370         skb = skb_share_check(skb, GFP_ATOMIC);
 371         if (skb == NULL)
 372                 goto oom;
 373
 374         /* drop any routing info */
 375         skb_dst_drop(skb);
 376
 377         /* drop conntrack reference */
 378         nf_reset(skb);
 379
 380         spkt = &PACKET_SKB_CB(skb)->sa.pkt;
 381
 382         skb_push(skb, skb->data - skb_mac_header(skb));
 383
 384         /*
 385          *      The SOCK_PACKET socket receives _all_ frames.
 386          */
 387
 388         spkt->spkt_family = dev->type;
 389         strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
 390         spkt->spkt_protocol = skb->protocol;
 391
 392         /*
 393          *      Charge the memory to the socket. This is done specifically
 394          *      to prevent sockets using all the memory up.
 395          */
 396
 397         if (sock_queue_rcv_skb(sk, skb) == 0)
 398                 return 0;
 399
 400 out:
 401         kfree_skb(skb);
 402 oom:
 403         return 0;
 404 }
 405
 406
 407 /*
 408  *      Output a raw packet to a device layer. This bypasses all the other
 409  *      protocol layers and you must therefore supply it with a complete frame
 410  */
 411
 412 static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
 413                                struct msghdr *msg, size_t len)
 414 {
 415         struct sock *sk = sock->sk;
 416         struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
 417         struct sk_buff *skb;
 418         struct net_device *dev;
 419         __be16 proto = 0;
 420         int err;
 421
 422         /*
 423          *      Get and verify the address.
 424          */
 425
 426         if (saddr) {
 427                 if (msg->msg_namelen < sizeof(struct sockaddr))
 428                         return -EINVAL;
 429                 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
 430                         proto = saddr->spkt_protocol;
 431         } else
 432                 return -ENOTCONN;       /* SOCK_PACKET must be sent giving an address */
 433
 434         /*
 435          *      Find the device first to size check it
 436          */
 437
 438         saddr->spkt_device[13] = 0;
 439         dev = dev_get_by_name(sock_net(sk), saddr->spkt_device);
 440         err = -ENODEV;
 441         if (dev == NULL)
 442                 goto out_unlock;
 443
 444         err = -ENETDOWN;
 445         if (!(dev->flags & IFF_UP))
 446                 goto out_unlock;
 447
 448         /*
 449          * You may not queue a frame bigger than the mtu. This is the lowest level
 450          * raw protocol and you must do your own fragmentation at this level.
 451          */
 452
 453         err = -EMSGSIZE;
 454         if (len > dev->mtu + dev->hard_header_len)
 455                 goto out_unlock;
 456
 457         err = -ENOBUFS;
 458         skb = sock_wmalloc(sk, len + LL_RESERVED_SPACE(dev), 0, GFP_KERNEL);
 459
 460         /*
 461          * If the write buffer is full, then tough. At this level the user
 462          * gets to deal with the problem - do your own algorithmic backoffs.
 463          * That's far more flexible.
 464          */
 465
 466         if (skb == NULL)
 467                 goto out_unlock;
 468
 469         /*
 470          *      Fill it in
 471          */
 472
 473         /* FIXME: Save some space for broken drivers that write a
 474          * hard header at transmission time by themselves. PPP is the
 475          * notable one here. This should really be fixed at the driver level.
 476          */
 477         skb_reserve(skb, LL_RESERVED_SPACE(dev));
 478         skb_reset_network_header(skb);
 479
 480         /* Try to align data part correctly */
 481         if (dev->header_ops) {
 482                 skb->data -= dev->hard_header_len;
 483                 skb->tail -= dev->hard_header_len;
 484                 if (len < dev->hard_header_len)
 485                         skb_reset_network_header(skb);
 486         }
 487
 488         /* Returns -EFAULT on error */
 489         err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
 490         skb->protocol = proto;
 491         skb->dev = dev;
 492         skb->priority = sk->sk_priority;
 493         if (err)
 494                 goto out_free;
 495
 496         /*
 497          *      Now send it
 498          */
 499
 500         dev_queue_xmit(skb);
 501         dev_put(dev);
 502         return len;
 503
 504 out_free:
 505         kfree_skb(skb);
 506 out_unlock:
 507         if (dev)
 508                 dev_put(dev);
 509         return err;
 510 }
 511
 512 static inline unsigned int run_filter(struct sk_buff *skb, struct sock *sk,
 513                                       unsigned int res)
 514 {
 515         struct sk_filter *filter;
 516
 517         rcu_read_lock_bh();
 518         filter = rcu_dereference(sk->sk_filter);
 519         if (filter != NULL)
 520                 res = sk_run_filter(skb, filter->insns, filter->len);
 521         rcu_read_unlock_bh();
 522
 523         return res;
 524 }
 525
 526 /*
 527    This function makes lazy skb cloning in hope that most of packets
 528    are discarded by BPF.
 529
 530    Note tricky part: we DO mangle shared skb! skb->data, skb->len
 531    and skb->cb are mangled. It works because (and until) packets
 532    falling here are owned by current CPU. Output packets are cloned
 533    by dev_queue_xmit_nit(), input packets are processed by net_bh
 534    sequencially, so that if we return skb to original state on exit,
 535    we will not harm anyone.
 536  */
 537
 538 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
 539                       struct packet_type *pt, struct net_device *orig_dev)
 540 {
 541         struct sock *sk;
 542         struct sockaddr_ll *sll;
 543         struct packet_sock *po;
 544         u8 *skb_head = skb->data;
 545         int skb_len = skb->len;
 546         unsigned int snaplen, res;
 547
 548         if (skb->pkt_type == PACKET_LOOPBACK)
 549                 goto drop;
 550
 551         sk = pt->af_packet_priv;
 552         po = pkt_sk(sk);
 553
 554         if (dev_net(dev) != sock_net(sk))
 555                 goto drop;
 556
 557         skb->dev = dev;
 558
 559         if (dev->header_ops) {
 560                 /* The device has an explicit notion of ll header,
 561                    exported to higher levels.
 562
 563                    Otherwise, the device hides datails of it frame
 564                    structure, so that corresponding packet head
 565                    never delivered to user.
 566                  */
 567                 if (sk->sk_type != SOCK_DGRAM)
 568                         skb_push(skb, skb->data - skb_mac_header(skb));
 569                 else if (skb->pkt_type == PACKET_OUTGOING) {
 570                         /* Special case: outgoing packets have ll header at head */
 571                         skb_pull(skb, skb_network_offset(skb));
 572                 }
 573         }
 574
 575         snaplen = skb->len;
 576
 577         res = run_filter(skb, sk, snaplen);
 578         if (!res)
 579                 goto drop_n_restore;
 580         if (snaplen > res)
 581                 snaplen = res;
 582
 583         if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
 584             (unsigned)sk->sk_rcvbuf)
 585                 goto drop_n_acct;
 586
 587         if (skb_shared(skb)) {
 588                 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
 589                 if (nskb == NULL)
 590                         goto drop_n_acct;
 591
 592                 if (skb_head != skb->data) {
 593                         skb->data = skb_head;
 594                         skb->len = skb_len;
 595                 }
 596                 kfree_skb(skb);
 597                 skb = nskb;
 598         }
 599
 600         BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
 601                      sizeof(skb->cb));
 602
 603         sll = &PACKET_SKB_CB(skb)->sa.ll;
 604         sll->sll_family = AF_PACKET;
 605         sll->sll_hatype = dev->type;
 606         sll->sll_protocol = skb->protocol;
 607         sll->sll_pkttype = skb->pkt_type;
 608         if (unlikely(po->origdev))
 609                 sll->sll_ifindex = orig_dev->ifindex;
 610         else
 611                 sll->sll_ifindex = dev->ifindex;
 612
 613         sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
 614
 615         PACKET_SKB_CB(skb)->origlen = skb->len;
 616
 617         if (pskb_trim(skb, snaplen))
 618                 goto drop_n_acct;
 619
 620         skb_set_owner_r(skb, sk);
 621         skb->dev = NULL;
 622         skb_dst_drop(skb);
 623
 624         /* drop conntrack reference */
 625         nf_reset(skb);
 626
 627         spin_lock(&sk->sk_receive_queue.lock);
 628         po->stats.tp_packets++;
 629         __skb_queue_tail(&sk->sk_receive_queue, skb);
 630         spin_unlock(&sk->sk_receive_queue.lock);
 631         sk->sk_data_ready(sk, skb->len);
 632         return 0;
 633
 634 drop_n_acct:
 635         spin_lock(&sk->sk_receive_queue.lock);
 636         po->stats.tp_drops++;
 637         spin_unlock(&sk->sk_receive_queue.lock);
 638
 639 drop_n_restore:
 640         if (skb_head != skb->data && skb_shared(skb)) {
 641                 skb->data = skb_head;
 642                 skb->len = skb_len;
 643         }
 644 drop:
 645         consume_skb(skb);
 646         return 0;
 647 }
 648
 649 #ifdef CONFIG_PACKET_MMAP
 650 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
 651                        struct packet_type *pt, struct net_device *orig_dev)
 652 {
 653         struct sock *sk;
 654         struct packet_sock *po;
 655         struct sockaddr_ll *sll;
 656         union {
 657                 struct tpacket_hdr *h1;
 658                 struct tpacket2_hdr *h2;
 659                 void *raw;
 660         } h;
 661         u8 *skb_head = skb->data;
 662         int skb_len = skb->len;
 663         unsigned int snaplen, res;
 664         unsigned long status = TP_STATUS_LOSING|TP_STATUS_USER;
 665         unsigned short macoff, netoff, hdrlen;
 666         struct sk_buff *copy_skb = NULL;
 667         struct timeval tv;
 668         struct timespec ts;
 669
 670         if (skb->pkt_type == PACKET_LOOPBACK)
 671                 goto drop;
 672
 673         sk = pt->af_packet_priv;
 674         po = pkt_sk(sk);
 675
 676         if (dev_net(dev) != sock_net(sk))
 677                 goto drop;
 678
 679         if (dev->header_ops) {
 680                 if (sk->sk_type != SOCK_DGRAM)
 681                         skb_push(skb, skb->data - skb_mac_header(skb));
 682                 else if (skb->pkt_type == PACKET_OUTGOING) {
 683                         /* Special case: outgoing packets have ll header at head */
 684                         skb_pull(skb, skb_network_offset(skb));
 685                 }
 686         }
 687
 688         if (skb->ip_summed == CHECKSUM_PARTIAL)
 689                 status |= TP_STATUS_CSUMNOTREADY;
 690
 691         snaplen = skb->len;
 692
 693         res = run_filter(skb, sk, snaplen);
 694         if (!res)
 695                 goto drop_n_restore;
 696         if (snaplen > res)
 697                 snaplen = res;
 698
 699         if (sk->sk_type == SOCK_DGRAM) {
 700                 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
 701                                   po->tp_reserve;
 702         } else {
 703                 unsigned maclen = skb_network_offset(skb);
 704                 netoff = TPACKET_ALIGN(po->tp_hdrlen +
 705                                        (maclen < 16 ? 16 : maclen)) +
 706                         po->tp_reserve;
 707                 macoff = netoff - maclen;
 708         }
 709
 710         if (macoff + snaplen > po->rx_ring.frame_size) {
 711                 if (po->copy_thresh &&
 712                     atomic_read(&sk->sk_rmem_alloc) + skb->truesize <
 713                     (unsigned)sk->sk_rcvbuf) {
 714                         if (skb_shared(skb)) {
 715                                 copy_skb = skb_clone(skb, GFP_ATOMIC);
 716                         } else {
 717                                 copy_skb = skb_get(skb);
 718                                 skb_head = skb->data;
 719                         }
 720                         if (copy_skb)
 721                                 skb_set_owner_r(copy_skb, sk);
 722                 }
 723                 snaplen = po->rx_ring.frame_size - macoff;
 724                 if ((int)snaplen < 0)
 725                         snaplen = 0;
 726         }
 727
 728         spin_lock(&sk->sk_receive_queue.lock);
 729         h.raw = packet_current_frame(po, &po->rx_ring, TP_STATUS_KERNEL);
 730         if (!h.raw)
 731                 goto ring_is_full;
 732         packet_increment_head(&po->rx_ring);
 733         po->stats.tp_packets++;
 734         if (copy_skb) {
 735                 status |= TP_STATUS_COPY;
 736                 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
 737         }
 738         if (!po->stats.tp_drops)
 739                 status &= ~TP_STATUS_LOSING;
 740         spin_unlock(&sk->sk_receive_queue.lock);
 741
 742         skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
 743
 744         switch (po->tp_version) {
 745         case TPACKET_V1:
 746                 h.h1->tp_len = skb->len;
 747                 h.h1->tp_snaplen = snaplen;
 748                 h.h1->tp_mac = macoff;
 749                 h.h1->tp_net = netoff;
 750                 if (skb->tstamp.tv64)
 751                         tv = ktime_to_timeval(skb->tstamp);
 752                 else
 753                         do_gettimeofday(&tv);
 754                 h.h1->tp_sec = tv.tv_sec;
 755                 h.h1->tp_usec = tv.tv_usec;
 756                 hdrlen = sizeof(*h.h1);
 757                 break;
 758         case TPACKET_V2:
 759                 h.h2->tp_len = skb->len;
 760                 h.h2->tp_snaplen = snaplen;
 761                 h.h2->tp_mac = macoff;
 762                 h.h2->tp_net = netoff;
 763                 if (skb->tstamp.tv64)
 764                         ts = ktime_to_timespec(skb->tstamp);
 765                 else
 766                         getnstimeofday(&ts);
 767                 h.h2->tp_sec = ts.tv_sec;
 768                 h.h2->tp_nsec = ts.tv_nsec;
 769                 h.h2->tp_vlan_tci = skb->vlan_tci;
 770                 hdrlen = sizeof(*h.h2);
 771                 break;
 772         default:
 773                 BUG();
 774         }
 775
 776         sll = h.raw + TPACKET_ALIGN(hdrlen);
 777         sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
 778         sll->sll_family = AF_PACKET;
 779         sll->sll_hatype = dev->type;
 780         sll->sll_protocol = skb->protocol;
 781         sll->sll_pkttype = skb->pkt_type;
 782         if (unlikely(po->origdev))
 783                 sll->sll_ifindex = orig_dev->ifindex;
 784         else
 785                 sll->sll_ifindex = dev->ifindex;
 786
 787         __packet_set_status(po, h.raw, status);
 788         smp_mb();
 789         {
 790                 struct page *p_start, *p_end;
 791                 u8 *h_end = h.raw + macoff + snaplen - 1;
 792
 793                 p_start = virt_to_page(h.raw);
 794                 p_end = virt_to_page(h_end);
 795                 while (p_start <= p_end) {
 796                         flush_dcache_page(p_start);
 797                         p_start++;
 798                 }
 799         }
 800
 801         sk->sk_data_ready(sk, 0);
 802
 803 drop_n_restore:
 804         if (skb_head != skb->data && skb_shared(skb)) {
 805                 skb->data = skb_head;
 806                 skb->len = skb_len;
 807         }
 808 drop:
 809         kfree_skb(skb);
 810         return 0;
 811
 812 ring_is_full:
 813         po->stats.tp_drops++;
 814         spin_unlock(&sk->sk_receive_queue.lock);
 815
 816         sk->sk_data_ready(sk, 0);
 817         kfree_skb(copy_skb);
 818         goto drop_n_restore;
 819 }
 820
 821 static void tpacket_destruct_skb(struct sk_buff *skb)
 822 {
 823         struct packet_sock *po = pkt_sk(skb->sk);
 824         void *ph;
 825
 826         BUG_ON(skb == NULL);
 827
 828         if (likely(po->tx_ring.pg_vec)) {
 829                 ph = skb_shinfo(skb)->destructor_arg;
 830                 BUG_ON(__packet_get_status(po, ph) != TP_STATUS_SENDING);
 831                 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
 832                 atomic_dec(&po->tx_ring.pending);
 833                 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
 834         }
 835
 836         sock_wfree(skb);
 837 }
 838
 839 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
 840                 void *frame, struct net_device *dev, int size_max,
 841                 __be16 proto, unsigned char *addr)
 842 {
 843         union {
 844                 struct tpacket_hdr *h1;
 845                 struct tpacket2_hdr *h2;
 846                 void *raw;
 847         } ph;
 848         int to_write, offset, len, tp_len, nr_frags, len_max;
 849         struct socket *sock = po->sk.sk_socket;
 850         struct page *page;
 851         void *data;
 852         int err;
 853
 854         ph.raw = frame;
 855
 856         skb->protocol = proto;
 857         skb->dev = dev;
 858         skb->priority = po->sk.sk_priority;
 859         skb_shinfo(skb)->destructor_arg = ph.raw;
 860
 861         switch (po->tp_version) {
 862         case TPACKET_V2:
 863                 tp_len = ph.h2->tp_len;
 864                 break;
 865         default:
 866                 tp_len = ph.h1->tp_len;
 867                 break;
 868         }
 869         if (unlikely(tp_len > size_max)) {
 870                 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
 871                 return -EMSGSIZE;
 872         }
 873
 874         skb_reserve(skb, LL_RESERVED_SPACE(dev));
 875         skb_reset_network_header(skb);
 876
 877         data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
 878         to_write = tp_len;
 879
 880         if (sock->type == SOCK_DGRAM) {
 881                 err = dev_hard_header(skb, dev, ntohs(proto), addr,
 882                                 NULL, tp_len);
 883                 if (unlikely(err < 0))
 884                         return -EINVAL;
 885         } else if (dev->hard_header_len) {
 886                 /* net device doesn't like empty head */
 887                 if (unlikely(tp_len <= dev->hard_header_len)) {
 888                         pr_err("packet size is too short (%d < %d)\n",
 889                                tp_len, dev->hard_header_len);
 890                         return -EINVAL;
 891                 }
 892
 893                 skb_push(skb, dev->hard_header_len);
 894                 err = skb_store_bits(skb, 0, data,
 895                                 dev->hard_header_len);
 896                 if (unlikely(err))
 897                         return err;
 898
 899                 data += dev->hard_header_len;
 900                 to_write -= dev->hard_header_len;
 901         }
 902
 903         err = -EFAULT;
 904         page = virt_to_page(data);
 905         offset = offset_in_page(data);
 906         len_max = PAGE_SIZE - offset;
 907         len = ((to_write > len_max) ? len_max : to_write);
 908
 909         skb->data_len = to_write;
 910         skb->len += to_write;
 911         skb->truesize += to_write;
 912         atomic_add(to_write, &po->sk.sk_wmem_alloc);
 913
 914         while (likely(to_write)) {
 915                 nr_frags = skb_shinfo(skb)->nr_frags;
 916
 917                 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
 918                         pr_err("Packet exceed the number of skb frags(%lu)\n",
 919                                MAX_SKB_FRAGS);
 920                         return -EFAULT;
 921                 }
 922
 923                 flush_dcache_page(page);
 924                 get_page(page);
 925                 skb_fill_page_desc(skb,
 926                                 nr_frags,
 927                                 page++, offset, len);
 928                 to_write -= len;
 929                 offset = 0;
 930                 len_max = PAGE_SIZE;
 931                 len = ((to_write > len_max) ? len_max : to_write);
 932         }
 933
 934         return tp_len;
 935 }
 936
 937 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
 938 {
 939         struct socket *sock;
 940         struct sk_buff *skb;
 941         struct net_device *dev;
 942         __be16 proto;
 943         int ifindex, err, reserve = 0;
 944         void *ph;
 945         struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
 946         int tp_len, size_max;
 947         unsigned char *addr;
 948         int len_sum = 0;
 949         int status = 0;
 950
 951         sock = po->sk.sk_socket;
 952
 953         mutex_lock(&po->pg_vec_lock);
 954
 955         err = -EBUSY;
 956         if (saddr == NULL) {
 957                 ifindex = po->ifindex;
 958                 proto   = po->num;
 959                 addr    = NULL;
 960         } else {
 961                 err = -EINVAL;
 962                 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
 963                         goto out;
 964                 if (msg->msg_namelen < (saddr->sll_halen
 965                                         + offsetof(struct sockaddr_ll,
 966                                                 sll_addr)))
 967                         goto out;
 968                 ifindex = saddr->sll_ifindex;
 969                 proto   = saddr->sll_protocol;
 970                 addr    = saddr->sll_addr;
 971         }
 972
 973         dev = dev_get_by_index(sock_net(&po->sk), ifindex);
 974         err = -ENXIO;
 975         if (unlikely(dev == NULL))
 976                 goto out;
 977
 978         reserve = dev->hard_header_len;
 979
 980         err = -ENETDOWN;
 981         if (unlikely(!(dev->flags & IFF_UP)))
 982                 goto out_put;
 983
 984         size_max = po->tx_ring.frame_size
 985                 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
 986
 987         if (size_max > dev->mtu + reserve)
 988                 size_max = dev->mtu + reserve;
 989
 990         do {
 991                 ph = packet_current_frame(po, &po->tx_ring,
 992                                 TP_STATUS_SEND_REQUEST);
 993
 994                 if (unlikely(ph == NULL)) {
 995                         schedule();
 996                         continue;
 997                 }
 998
 999                 status = TP_STATUS_SEND_REQUEST;
1000                 skb = sock_alloc_send_skb(&po->sk,
1001                                 LL_ALLOCATED_SPACE(dev)
1002                                 + sizeof(struct sockaddr_ll),
1003                                 0, &err);
1004
1005                 if (unlikely(skb == NULL))
1006                         goto out_status;
1007
1008                 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
1009                                 addr);
1010
1011                 if (unlikely(tp_len < 0)) {
1012                         if (po->tp_loss) {
1013                                 __packet_set_status(po, ph,
1014                                                 TP_STATUS_AVAILABLE);
1015                                 packet_increment_head(&po->tx_ring);
1016                                 kfree_skb(skb);
1017                                 continue;
1018                         } else {
1019                                 status = TP_STATUS_WRONG_FORMAT;
1020                                 err = tp_len;
1021                                 goto out_status;
1022                         }
1023                 }
1024
1025                 skb->destructor = tpacket_destruct_skb;
1026                 __packet_set_status(po, ph, TP_STATUS_SENDING);
1027                 atomic_inc(&po->tx_ring.pending);
1028
1029                 status = TP_STATUS_SEND_REQUEST;
1030                 err = dev_queue_xmit(skb);
1031                 if (unlikely(err > 0 && (err = net_xmit_errno(err)) != 0))
1032                         goto out_xmit;
1033                 packet_increment_head(&po->tx_ring);
1034                 len_sum += tp_len;
1035         } while (likely((ph != NULL) || ((!(msg->msg_flags & MSG_DONTWAIT))
1036                                         && (atomic_read(&po->tx_ring.pending))))
1037               );
1038
1039         err = len_sum;
1040         goto out_put;
1041
1042 out_xmit:
1043         skb->destructor = sock_wfree;
1044         atomic_dec(&po->tx_ring.pending);
1045 out_status:
1046         __packet_set_status(po, ph, status);
1047         kfree_skb(skb);
1048 out_put:
1049         dev_put(dev);
1050 out:
1051         mutex_unlock(&po->pg_vec_lock);
1052         return err;
1053 }
1054 #endif
1055
1056 static int packet_snd(struct socket *sock,
1057                           struct msghdr *msg, size_t len)
1058 {
1059         struct sock *sk = sock->sk;
1060         struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1061         struct sk_buff *skb;
1062         struct net_device *dev;
1063         __be16 proto;
1064         unsigned char *addr;
1065         int ifindex, err, reserve = 0;
1066
1067         /*
1068          *      Get and verify the address.
1069          */
1070
1071         if (saddr == NULL) {
1072                 struct packet_sock *po = pkt_sk(sk);
1073
1074                 ifindex = po->ifindex;
1075                 proto   = po->num;
1076                 addr    = NULL;
1077         } else {
1078                 err = -EINVAL;
1079                 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
1080                         goto out;
1081                 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
1082                         goto out;
1083                 ifindex = saddr->sll_ifindex;
1084                 proto   = saddr->sll_protocol;
1085                 addr    = saddr->sll_addr;
1086         }
1087
1088
1089         dev = dev_get_by_index(sock_net(sk), ifindex);
1090         err = -ENXIO;
1091         if (dev == NULL)
1092                 goto out_unlock;
1093         if (sock->type == SOCK_RAW)
1094                 reserve = dev->hard_header_len;
1095
1096         err = -ENETDOWN;
1097         if (!(dev->flags & IFF_UP))
1098                 goto out_unlock;
1099
1100         err = -EMSGSIZE;
1101         if (len > dev->mtu+reserve)
1102                 goto out_unlock;
1103
1104         skb = sock_alloc_send_skb(sk, len + LL_ALLOCATED_SPACE(dev),
1105                                 msg->msg_flags & MSG_DONTWAIT, &err);
1106         if (skb == NULL)
1107                 goto out_unlock;
1108
1109         skb_reserve(skb, LL_RESERVED_SPACE(dev));
1110         skb_reset_network_header(skb);
1111
1112         err = -EINVAL;
1113         if (sock->type == SOCK_DGRAM &&
1114             dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len) < 0)
1115                 goto out_free;
1116
1117         /* Returns -EFAULT on error */
1118         err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1119         if (err)
1120                 goto out_free;
1121
1122         skb->protocol = proto;
1123         skb->dev = dev;
1124         skb->priority = sk->sk_priority;
1125
1126         /*
1127          *      Now send it
1128          */
1129
1130         err = dev_queue_xmit(skb);
1131         if (err > 0 && (err = net_xmit_errno(err)) != 0)
1132                 goto out_unlock;
1133
1134         dev_put(dev);
1135
1136         return len;
1137
1138 out_free:
1139         kfree_skb(skb);
1140 out_unlock:
1141         if (dev)
1142                 dev_put(dev);
1143 out:
1144         return err;
1145 }
1146
1147 static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
1148                 struct msghdr *msg, size_t len)
1149 {
1150 #ifdef CONFIG_PACKET_MMAP
1151         struct sock *sk = sock->sk;
1152         struct packet_sock *po = pkt_sk(sk);
1153         if (po->tx_ring.pg_vec)
1154                 return tpacket_snd(po, msg);
1155         else
1156 #endif
1157                 return packet_snd(sock, msg, len);
1158 }
1159
1160 /*
1161  *      Close a PACKET socket. This is fairly simple. We immediately go
1162  *      to 'closed' state and remove our protocol entry in the device list.
1163  */
1164
1165 static int packet_release(struct socket *sock)
1166 {
1167         struct sock *sk = sock->sk;
1168         struct packet_sock *po;
1169         struct net *net;
1170 #ifdef CONFIG_PACKET_MMAP
1171         struct tpacket_req req;
1172 #endif
1173
1174         if (!sk)
1175                 return 0;
1176
1177         net = sock_net(sk);
1178         po = pkt_sk(sk);
1179
1180         write_lock_bh(&net->packet.sklist_lock);
1181         sk_del_node_init(sk);
1182         sock_prot_inuse_add(net, sk->sk_prot, -1);
1183         write_unlock_bh(&net->packet.sklist_lock);
1184
1185         /*
1186          *      Unhook packet receive handler.
1187          */
1188
1189         if (po->running) {
1190                 /*
1191                  *      Remove the protocol hook
1192                  */
1193                 dev_remove_pack(&po->prot_hook);
1194                 po->running = 0;
1195                 po->num = 0;
1196                 __sock_put(sk);
1197         }
1198
1199         packet_flush_mclist(sk);
1200
1201 #ifdef CONFIG_PACKET_MMAP
1202         memset(&req, 0, sizeof(req));
1203
1204         if (po->rx_ring.pg_vec)
1205                 packet_set_ring(sk, &req, 1, 0);
1206
1207         if (po->tx_ring.pg_vec)
1208                 packet_set_ring(sk, &req, 1, 1);
1209 #endif
1210
1211         /*
1212          *      Now the socket is dead. No more input will appear.
1213          */
1214
1215         sock_orphan(sk);
1216         sock->sk = NULL;
1217
1218         /* Purge queues */
1219
1220         skb_queue_purge(&sk->sk_receive_queue);
1221         sk_refcnt_debug_release(sk);
1222
1223         sock_put(sk);
1224         return 0;
1225 }
1226
1227 /*
1228  *      Attach a packet hook.
1229  */
1230
1231 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1232 {
1233         struct packet_sock *po = pkt_sk(sk);
1234         /*
1235          *      Detach an existing hook if present.
1236          */
1237
1238         lock_sock(sk);
1239
1240         spin_lock(&po->bind_lock);
1241         if (po->running) {
1242                 __sock_put(sk);
1243                 po->running = 0;
1244                 po->num = 0;
1245                 spin_unlock(&po->bind_lock);
1246                 dev_remove_pack(&po->prot_hook);
1247                 spin_lock(&po->bind_lock);
1248         }
1249
1250         po->num = protocol;
1251         po->prot_hook.type = protocol;
1252         po->prot_hook.dev = dev;
1253
1254         po->ifindex = dev ? dev->ifindex : 0;
1255
1256         if (protocol == 0)
1257                 goto out_unlock;
1258
1259         if (!dev || (dev->flags & IFF_UP)) {
1260                 dev_add_pack(&po->prot_hook);
1261                 sock_hold(sk);
1262                 po->running = 1;
1263         } else {
1264                 sk->sk_err = ENETDOWN;
1265                 if (!sock_flag(sk, SOCK_DEAD))
1266                         sk->sk_error_report(sk);
1267         }
1268
1269 out_unlock:
1270         spin_unlock(&po->bind_lock);
1271         release_sock(sk);
1272         return 0;
1273 }
1274
1275 /*
1276  *      Bind a packet socket to a device
1277  */
1278
1279 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
1280                             int addr_len)
1281 {
1282         struct sock *sk = sock->sk;
1283         char name[15];
1284         struct net_device *dev;
1285         int err = -ENODEV;
1286
1287         /*
1288          *      Check legality
1289          */
1290
1291         if (addr_len != sizeof(struct sockaddr))
1292                 return -EINVAL;
1293         strlcpy(name, uaddr->sa_data, sizeof(name));
1294
1295         dev = dev_get_by_name(sock_net(sk), name);
1296         if (dev) {
1297                 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1298                 dev_put(dev);
1299         }
1300         return err;
1301 }
1302
1303 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
1304 {
1305         struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
1306         struct sock *sk = sock->sk;
1307         struct net_device *dev = NULL;
1308         int err;
1309
1310
1311         /*
1312          *      Check legality
1313          */
1314
1315         if (addr_len < sizeof(struct sockaddr_ll))
1316                 return -EINVAL;
1317         if (sll->sll_family != AF_PACKET)
1318                 return -EINVAL;
1319
1320         if (sll->sll_ifindex) {
1321                 err = -ENODEV;
1322                 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1323                 if (dev == NULL)
1324                         goto out;
1325         }
1326         err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1327         if (dev)
1328                 dev_put(dev);
1329
1330 out:
1331         return err;
1332 }
1333
1334 static struct proto packet_proto = {
1335         .name     = "PACKET",
1336         .owner    = THIS_MODULE,
1337         .obj_size = sizeof(struct packet_sock),
1338 };
1339
1340 /*
1341  *      Create a packet of type SOCK_PACKET.
1342  */
1343
1344 static int packet_create(struct net *net, struct socket *sock, int protocol)
1345 {
1346         struct sock *sk;
1347         struct packet_sock *po;
1348         __be16 proto = (__force __be16)protocol; /* weird, but documented */
1349         int err;
1350
1351         if (!capable(CAP_NET_RAW))
1352                 return -EPERM;
1353         if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
1354             sock->type != SOCK_PACKET)
1355                 return -ESOCKTNOSUPPORT;
1356
1357         sock->state = SS_UNCONNECTED;
1358
1359         err = -ENOBUFS;
1360         sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1361         if (sk == NULL)
1362                 goto out;
1363
1364         sock->ops = &packet_ops;
1365         if (sock->type == SOCK_PACKET)
1366                 sock->ops = &packet_ops_spkt;
1367
1368         sock_init_data(sock, sk);
1369
1370         po = pkt_sk(sk);
1371         sk->sk_family = PF_PACKET;
1372         po->num = proto;
1373
1374         sk->sk_destruct = packet_sock_destruct;
1375         sk_refcnt_debug_inc(sk);
1376
1377         /*
1378          *      Attach a protocol block
1379          */
1380
1381         spin_lock_init(&po->bind_lock);
1382         mutex_init(&po->pg_vec_lock);
1383         po->prot_hook.func = packet_rcv;
1384
1385         if (sock->type == SOCK_PACKET)
1386                 po->prot_hook.func = packet_rcv_spkt;
1387
1388         po->prot_hook.af_packet_priv = sk;
1389
1390         if (proto) {
1391                 po->prot_hook.type = proto;
1392                 dev_add_pack(&po->prot_hook);
1393                 sock_hold(sk);
1394                 po->running = 1;
1395         }
1396
1397         write_lock_bh(&net->packet.sklist_lock);
1398         sk_add_node(sk, &net->packet.sklist);
1399         sock_prot_inuse_add(net, &packet_proto, 1);
1400         write_unlock_bh(&net->packet.sklist_lock);
1401         return 0;
1402 out:
1403         return err;
1404 }
1405
1406 /*
1407  *      Pull a packet from our receive queue and hand it to the user.
1408  *      If necessary we block.
1409  */
1410
1411 static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
1412                           struct msghdr *msg, size_t len, int flags)
1413 {
1414         struct sock *sk = sock->sk;
1415         struct sk_buff *skb;
1416         int copied, err;
1417         struct sockaddr_ll *sll;
1418
1419         err = -EINVAL;
1420         if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT))
1421                 goto out;
1422
1423 #if 0
1424         /* What error should we return now? EUNATTACH? */
1425         if (pkt_sk(sk)->ifindex < 0)
1426                 return -ENODEV;
1427 #endif
1428
1429         /*
1430          *      Call the generic datagram receiver. This handles all sorts
1431          *      of horrible races and re-entrancy so we can forget about it
1432          *      in the protocol layers.
1433          *
1434          *      Now it will return ENETDOWN, if device have just gone down,
1435          *      but then it will block.
1436          */
1437
1438         skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1439
1440         /*
1441          *      An error occurred so return it. Because skb_recv_datagram()
1442          *      handles the blocking we don't see and worry about blocking
1443          *      retries.
1444          */
1445
1446         if (skb == NULL)
1447                 goto out;
1448
1449         /*
1450          *      If the address length field is there to be filled in, we fill
1451          *      it in now.
1452          */
1453
1454         sll = &PACKET_SKB_CB(skb)->sa.ll;
1455         if (sock->type == SOCK_PACKET)
1456                 msg->msg_namelen = sizeof(struct sockaddr_pkt);
1457         else
1458                 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
1459
1460         /*
1461          *      You lose any data beyond the buffer you gave. If it worries a
1462          *      user program they can ask the device for its MTU anyway.
1463          */
1464
1465         copied = skb->len;
1466         if (copied > len) {
1467                 copied = len;
1468                 msg->msg_flags |= MSG_TRUNC;
1469         }
1470
1471         err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
1472         if (err)
1473                 goto out_free;
1474
1475         sock_recv_timestamp(msg, sk, skb);
1476
1477         if (msg->msg_name)
1478                 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
1479                        msg->msg_namelen);
1480
1481         if (pkt_sk(sk)->auxdata) {
1482                 struct tpacket_auxdata aux;
1483
1484                 aux.tp_status = TP_STATUS_USER;
1485                 if (skb->ip_summed == CHECKSUM_PARTIAL)
1486                         aux.tp_status |= TP_STATUS_CSUMNOTREADY;
1487                 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
1488                 aux.tp_snaplen = skb->len;
1489                 aux.tp_mac = 0;
1490                 aux.tp_net = skb_network_offset(skb);
1491                 aux.tp_vlan_tci = skb->vlan_tci;
1492
1493                 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
1494         }
1495
1496         /*
1497          *      Free or return the buffer as appropriate. Again this
1498          *      hides all the races and re-entrancy issues from us.
1499          */
1500         err = (flags&MSG_TRUNC) ? skb->len : copied;
1501
1502 out_free:
1503         skb_free_datagram(sk, skb);
1504 out:
1505         return err;
1506 }
1507
1508 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
1509                                int *uaddr_len, int peer)
1510 {
1511         struct net_device *dev;
1512         struct sock *sk = sock->sk;
1513
1514         if (peer)
1515                 return -EOPNOTSUPP;
1516
1517         uaddr->sa_family = AF_PACKET;
1518         dev = dev_get_by_index(sock_net(sk), pkt_sk(sk)->ifindex);
1519         if (dev) {
1520                 strlcpy(uaddr->sa_data, dev->name, 15);
1521                 dev_put(dev);
1522         } else
1523                 memset(uaddr->sa_data, 0, 14);
1524         *uaddr_len = sizeof(*uaddr);
1525
1526         return 0;
1527 }
1528
1529 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
1530                           int *uaddr_len, int peer)
1531 {
1532         struct net_device *dev;
1533         struct sock *sk = sock->sk;
1534         struct packet_sock *po = pkt_sk(sk);
1535         struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
1536
1537         if (peer)
1538                 return -EOPNOTSUPP;
1539
1540         sll->sll_family = AF_PACKET;
1541         sll->sll_ifindex = po->ifindex;
1542         sll->sll_protocol = po->num;
1543         dev = dev_get_by_index(sock_net(sk), po->ifindex);
1544         if (dev) {
1545                 sll->sll_hatype = dev->type;
1546                 sll->sll_halen = dev->addr_len;
1547                 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1548                 dev_put(dev);
1549         } else {
1550                 sll->sll_hatype = 0;    /* Bad: we have no ARPHRD_UNSPEC */
1551                 sll->sll_halen = 0;
1552         }
1553         *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1554
1555         return 0;
1556 }
1557
1558 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
1559                          int what)
1560 {
1561         switch (i->type) {
1562         case PACKET_MR_MULTICAST:
1563                 if (what > 0)
1564                         return dev_mc_add(dev, i->addr, i->alen, 0);
1565                 else
1566                         return dev_mc_delete(dev, i->addr, i->alen, 0);
1567                 break;
1568         case PACKET_MR_PROMISC:
1569                 return dev_set_promiscuity(dev, what);
1570                 break;
1571         case PACKET_MR_ALLMULTI:
1572                 return dev_set_allmulti(dev, what);
1573                 break;
1574         case PACKET_MR_UNICAST:
1575                 if (what > 0)
1576                         return dev_unicast_add(dev, i->addr);
1577                 else
1578                         return dev_unicast_delete(dev, i->addr);
1579                 break;
1580         default:
1581                 break;
1582         }
1583         return 0;
1584 }
1585
1586 static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
1587 {
1588         for ( ; i; i = i->next) {
1589                 if (i->ifindex == dev->ifindex)
1590                         packet_dev_mc(dev, i, what);
1591         }
1592 }
1593
1594 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1595 {
1596         struct packet_sock *po = pkt_sk(sk);
1597         struct packet_mclist *ml, *i;
1598         struct net_device *dev;
1599         int err;
1600
1601         rtnl_lock();
1602
1603         err = -ENODEV;
1604         dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1605         if (!dev)
1606                 goto done;
1607
1608         err = -EINVAL;
1609         if (mreq->mr_alen > dev->addr_len)
1610                 goto done;
1611
1612         err = -ENOBUFS;
1613         i = kmalloc(sizeof(*i), GFP_KERNEL);
1614         if (i == NULL)
1615                 goto done;
1616
1617         err = 0;
1618         for (ml = po->mclist; ml; ml = ml->next) {
1619                 if (ml->ifindex == mreq->mr_ifindex &&
1620                     ml->type == mreq->mr_type &&
1621                     ml->alen == mreq->mr_alen &&
1622                     memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
1623                         ml->count++;
1624                         /* Free the new element ... */
1625                         kfree(i);
1626                         goto done;
1627                 }
1628         }
1629
1630         i->type = mreq->mr_type;
1631         i->ifindex = mreq->mr_ifindex;
1632         i->alen = mreq->mr_alen;
1633         memcpy(i->addr, mreq->mr_address, i->alen);
1634         i->count = 1;
1635         i->next = po->mclist;
1636         po->mclist = i;
1637         err = packet_dev_mc(dev, i, 1);
1638         if (err) {
1639                 po->mclist = i->next;
1640                 kfree(i);
1641         }
1642
1643 done:
1644         rtnl_unlock();
1645         return err;
1646 }
1647
1648 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1649 {
1650         struct packet_mclist *ml, **mlp;
1651
1652         rtnl_lock();
1653
1654         for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
1655                 if (ml->ifindex == mreq->mr_ifindex &&
1656                     ml->type == mreq->mr_type &&
1657                     ml->alen == mreq->mr_alen &&
1658                     memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
1659                         if (--ml->count == 0) {
1660                                 struct net_device *dev;
1661                                 *mlp = ml->next;
1662                                 dev = dev_get_by_index(sock_net(sk), ml->ifindex);
1663                                 if (dev) {
1664                                         packet_dev_mc(dev, ml, -1);
1665                                         dev_put(dev);
1666                                 }
1667                                 kfree(ml);
1668                         }
1669                         rtnl_unlock();
1670                         return 0;
1671                 }
1672         }
1673         rtnl_unlock();
1674         return -EADDRNOTAVAIL;
1675 }
1676
1677 static void packet_flush_mclist(struct sock *sk)
1678 {
1679         struct packet_sock *po = pkt_sk(sk);
1680         struct packet_mclist *ml;
1681
1682         if (!po->mclist)
1683                 return;
1684
1685         rtnl_lock();
1686         while ((ml = po->mclist) != NULL) {
1687                 struct net_device *dev;
1688
1689                 po->mclist = ml->next;
1690                 dev = dev_get_by_index(sock_net(sk), ml->ifindex);
1691                 if (dev != NULL) {
1692                         packet_dev_mc(dev, ml, -1);
1693                         dev_put(dev);
1694                 }
1695                 kfree(ml);
1696         }
1697         rtnl_unlock();
1698 }
1699
1700 static int
1701 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1702 {
1703         struct sock *sk = sock->sk;
1704         struct packet_sock *po = pkt_sk(sk);
1705         int ret;
1706
1707         if (level != SOL_PACKET)
1708                 return -ENOPROTOOPT;
1709
1710         switch (optname) {
1711         case PACKET_ADD_MEMBERSHIP:
1712         case PACKET_DROP_MEMBERSHIP:
1713         {
1714                 struct packet_mreq_max mreq;
1715                 int len = optlen;
1716                 memset(&mreq, 0, sizeof(mreq));
1717                 if (len < sizeof(struct packet_mreq))
1718                         return -EINVAL;
1719                 if (len > sizeof(mreq))
1720                         len = sizeof(mreq);
1721                 if (copy_from_user(&mreq, optval, len))
1722                         return -EFAULT;
1723                 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
1724                         return -EINVAL;
1725                 if (optname == PACKET_ADD_MEMBERSHIP)
1726                         ret = packet_mc_add(sk, &mreq);
1727                 else
1728                         ret = packet_mc_drop(sk, &mreq);
1729                 return ret;
1730         }
1731
1732 #ifdef CONFIG_PACKET_MMAP
1733         case PACKET_RX_RING:
1734         case PACKET_TX_RING:
1735         {
1736                 struct tpacket_req req;
1737
1738                 if (optlen < sizeof(req))
1739                         return -EINVAL;
1740                 if (copy_from_user(&req, optval, sizeof(req)))
1741                         return -EFAULT;
1742                 return packet_set_ring(sk, &req, 0, optname == PACKET_TX_RING);
1743         }
1744         case PACKET_COPY_THRESH:
1745         {
1746                 int val;
1747
1748                 if (optlen != sizeof(val))
1749                         return -EINVAL;
1750                 if (copy_from_user(&val, optval, sizeof(val)))
1751                         return -EFAULT;
1752
1753                 pkt_sk(sk)->copy_thresh = val;
1754                 return 0;
1755         }
1756         case PACKET_VERSION:
1757         {
1758                 int val;
1759
1760                 if (optlen != sizeof(val))
1761                         return -EINVAL;
1762                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1763                         return -EBUSY;
1764                 if (copy_from_user(&val, optval, sizeof(val)))
1765                         return -EFAULT;
1766                 switch (val) {
1767                 case TPACKET_V1:
1768                 case TPACKET_V2:
1769                         po->tp_version = val;
1770                         return 0;
1771                 default:
1772                         return -EINVAL;
1773                 }
1774         }
1775         case PACKET_RESERVE:
1776         {
1777                 unsigned int val;
1778
1779                 if (optlen != sizeof(val))
1780                         return -EINVAL;
1781                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1782                         return -EBUSY;
1783                 if (copy_from_user(&val, optval, sizeof(val)))
1784                         return -EFAULT;
1785                 po->tp_reserve = val;
1786                 return 0;
1787         }
1788         case PACKET_LOSS:
1789         {
1790                 unsigned int val;
1791
1792                 if (optlen != sizeof(val))
1793                         return -EINVAL;
1794                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1795                         return -EBUSY;
1796                 if (copy_from_user(&val, optval, sizeof(val)))
1797                         return -EFAULT;
1798                 po->tp_loss = !!val;
1799                 return 0;
1800         }
1801 #endif
1802         case PACKET_AUXDATA:
1803         {
1804                 int val;
1805
1806                 if (optlen < sizeof(val))
1807                         return -EINVAL;
1808                 if (copy_from_user(&val, optval, sizeof(val)))
1809                         return -EFAULT;
1810
1811                 po->auxdata = !!val;
1812                 return 0;
1813         }
1814         case PACKET_ORIGDEV:
1815         {
1816                 int val;
1817
1818                 if (optlen < sizeof(val))
1819                         return -EINVAL;
1820                 if (copy_from_user(&val, optval, sizeof(val)))
1821                         return -EFAULT;
1822
1823                 po->origdev = !!val;
1824                 return 0;
1825         }
1826         default:
1827                 return -ENOPROTOOPT;
1828         }
1829 }
1830
1831 static int packet_getsockopt(struct socket *sock, int level, int optname,
1832                              char __user *optval, int __user *optlen)
1833 {
1834         int len;
1835         int val;
1836         struct sock *sk = sock->sk;
1837         struct packet_sock *po = pkt_sk(sk);
1838         void *data;
1839         struct tpacket_stats st;
1840
1841         if (level != SOL_PACKET)
1842                 return -ENOPROTOOPT;
1843
1844         if (get_user(len, optlen))
1845                 return -EFAULT;
1846
1847         if (len < 0)
1848                 return -EINVAL;
1849
1850         switch (optname) {
1851         case PACKET_STATISTICS:
1852                 if (len > sizeof(struct tpacket_stats))
1853                         len = sizeof(struct tpacket_stats);
1854                 spin_lock_bh(&sk->sk_receive_queue.lock);
1855                 st = po->stats;
1856                 memset(&po->stats, 0, sizeof(st));
1857                 spin_unlock_bh(&sk->sk_receive_queue.lock);
1858                 st.tp_packets += st.tp_drops;
1859
1860                 data = &st;
1861                 break;
1862         case PACKET_AUXDATA:
1863                 if (len > sizeof(int))
1864                         len = sizeof(int);
1865                 val = po->auxdata;
1866
1867                 data = &val;
1868                 break;
1869         case PACKET_ORIGDEV:
1870                 if (len > sizeof(int))
1871                         len = sizeof(int);
1872                 val = po->origdev;
1873
1874                 data = &val;
1875                 break;
1876 #ifdef CONFIG_PACKET_MMAP
1877         case PACKET_VERSION:
1878                 if (len > sizeof(int))
1879                         len = sizeof(int);
1880                 val = po->tp_version;
1881                 data = &val;
1882                 break;
1883         case PACKET_HDRLEN:
1884                 if (len > sizeof(int))
1885                         len = sizeof(int);
1886                 if (copy_from_user(&val, optval, len))
1887                         return -EFAULT;
1888                 switch (val) {
1889                 case TPACKET_V1:
1890                         val = sizeof(struct tpacket_hdr);
1891                         break;
1892                 case TPACKET_V2:
1893                         val = sizeof(struct tpacket2_hdr);
1894                         break;
1895                 default:
1896                         return -EINVAL;
1897                 }
1898                 data = &val;
1899                 break;
1900         case PACKET_RESERVE:
1901                 if (len > sizeof(unsigned int))
1902                         len = sizeof(unsigned int);
1903                 val = po->tp_reserve;
1904                 data = &val;
1905                 break;
1906         case PACKET_LOSS:
1907                 if (len > sizeof(unsigned int))
1908                         len = sizeof(unsigned int);
1909                 val = po->tp_loss;
1910                 data = &val;
1911                 break;
1912 #endif
1913         default:
1914                 return -ENOPROTOOPT;
1915         }
1916
1917         if (put_user(len, optlen))
1918                 return -EFAULT;
1919         if (copy_to_user(optval, data, len))
1920                 return -EFAULT;
1921         return 0;
1922 }
1923
1924
1925 static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
1926 {
1927         struct sock *sk;
1928         struct hlist_node *node;
1929         struct net_device *dev = data;
1930         struct net *net = dev_net(dev);
1931
1932         read_lock(&net->packet.sklist_lock);
1933         sk_for_each(sk, node, &net->packet.sklist) {
1934                 struct packet_sock *po = pkt_sk(sk);
1935
1936                 switch (msg) {
1937                 case NETDEV_UNREGISTER:
1938                         if (po->mclist)
1939                                 packet_dev_mclist(dev, po->mclist, -1);
1940                         /* fallthrough */
1941
1942                 case NETDEV_DOWN:
1943                         if (dev->ifindex == po->ifindex) {
1944                                 spin_lock(&po->bind_lock);
1945                                 if (po->running) {
1946                                         __dev_remove_pack(&po->prot_hook);
1947                                         __sock_put(sk);
1948                                         po->running = 0;
1949                                         sk->sk_err = ENETDOWN;
1950                                         if (!sock_flag(sk, SOCK_DEAD))
1951                                                 sk->sk_error_report(sk);
1952                                 }
1953                                 if (msg == NETDEV_UNREGISTER) {
1954                                         po->ifindex = -1;
1955                                         po->prot_hook.dev = NULL;
1956                                 }
1957                                 spin_unlock(&po->bind_lock);
1958                         }
1959                         break;
1960                 case NETDEV_UP:
1961                         spin_lock(&po->bind_lock);
1962                         if (dev->ifindex == po->ifindex && po->num &&
1963                             !po->running) {
1964                                 dev_add_pack(&po->prot_hook);
1965                                 sock_hold(sk);
1966                                 po->running = 1;
1967                         }
1968                         spin_unlock(&po->bind_lock);
1969                         break;
1970                 }
1971         }
1972         read_unlock(&net->packet.sklist_lock);
1973         return NOTIFY_DONE;
1974 }
1975
1976
1977 static int packet_ioctl(struct socket *sock, unsigned int cmd,
1978                         unsigned long arg)
1979 {
1980         struct sock *sk = sock->sk;
1981
1982         switch (cmd) {
1983         case SIOCOUTQ:
1984         {
1985                 int amount = sk_wmem_alloc_get(sk);
1986
1987                 return put_user(amount, (int __user *)arg);
1988         }
1989         case SIOCINQ:
1990         {
1991                 struct sk_buff *skb;
1992                 int amount = 0;
1993
1994                 spin_lock_bh(&sk->sk_receive_queue.lock);
1995                 skb = skb_peek(&sk->sk_receive_queue);
1996                 if (skb)
1997                         amount = skb->len;
1998                 spin_unlock_bh(&sk->sk_receive_queue.lock);
1999                 return put_user(amount, (int __user *)arg);
2000         }
2001         case SIOCGSTAMP:
2002                 return sock_get_timestamp(sk, (struct timeval __user *)arg);
2003         case SIOCGSTAMPNS:
2004                 return sock_get_timestampns(sk, (struct timespec __user *)arg);
2005
2006 #ifdef CONFIG_INET
2007         case SIOCADDRT:
2008         case SIOCDELRT:
2009         case SIOCDARP:
2010         case SIOCGARP:
2011         case SIOCSARP:
2012         case SIOCGIFADDR:
2013         case SIOCSIFADDR:
2014         case SIOCGIFBRDADDR:
2015         case SIOCSIFBRDADDR:
2016         case SIOCGIFNETMASK:
2017         case SIOCSIFNETMASK:
2018         case SIOCGIFDSTADDR:
2019         case SIOCSIFDSTADDR:
2020         case SIOCSIFFLAGS:
2021                 if (!net_eq(sock_net(sk), &init_net))
2022                         return -ENOIOCTLCMD;
2023                 return inet_dgram_ops.ioctl(sock, cmd, arg);
2024 #endif
2025
2026         default:
2027                 return -ENOIOCTLCMD;
2028         }
2029         return 0;
2030 }
2031
2032 #ifndef CONFIG_PACKET_MMAP
2033 #define packet_mmap sock_no_mmap
2034 #define packet_poll datagram_poll
2035 #else
2036
2037 static unsigned int packet_poll(struct file *file, struct socket *sock,
2038                                 poll_table *wait)
2039 {
2040         struct sock *sk = sock->sk;
2041         struct packet_sock *po = pkt_sk(sk);
2042         unsigned int mask = datagram_poll(file, sock, wait);
2043
2044         spin_lock_bh(&sk->sk_receive_queue.lock);
2045         if (po->rx_ring.pg_vec) {
2046                 if (!packet_previous_frame(po, &po->rx_ring, TP_STATUS_KERNEL))
2047                         mask |= POLLIN | POLLRDNORM;
2048         }
2049         spin_unlock_bh(&sk->sk_receive_queue.lock);
2050         spin_lock_bh(&sk->sk_write_queue.lock);
2051         if (po->tx_ring.pg_vec) {
2052                 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
2053                         mask |= POLLOUT | POLLWRNORM;
2054         }
2055         spin_unlock_bh(&sk->sk_write_queue.lock);
2056         return mask;
2057 }
2058
2059
2060 /* Dirty? Well, I still did not learn better way to account
2061  * for user mmaps.
2062  */
2063
2064 static void packet_mm_open(struct vm_area_struct *vma)
2065 {
2066         struct file *file = vma->vm_file;
2067         struct socket *sock = file->private_data;
2068         struct sock *sk = sock->sk;
2069
2070         if (sk)
2071                 atomic_inc(&pkt_sk(sk)->mapped);
2072 }
2073
2074 static void packet_mm_close(struct vm_area_struct *vma)
2075 {
2076         struct file *file = vma->vm_file;
2077         struct socket *sock = file->private_data;
2078         struct sock *sk = sock->sk;
2079
2080         if (sk)
2081                 atomic_dec(&pkt_sk(sk)->mapped);
2082 }
2083
2084 static const struct vm_operations_struct packet_mmap_ops = {
2085         .open   =       packet_mm_open,
2086         .close  =       packet_mm_close,
2087 };
2088
2089 static void free_pg_vec(char **pg_vec, unsigned int order, unsigned int len)
2090 {
2091         int i;
2092
2093         for (i = 0; i < len; i++) {
2094                 if (likely(pg_vec[i]))
2095                         free_pages((unsigned long) pg_vec[i], order);
2096         }
2097         kfree(pg_vec);
2098 }
2099
2100 static inline char *alloc_one_pg_vec_page(unsigned long order)
2101 {
2102         gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | __GFP_ZERO | __GFP_NOWARN;
2103
2104         return (char *) __get_free_pages(gfp_flags, order);
2105 }
2106
2107 static char **alloc_pg_vec(struct tpacket_req *req, int order)
2108 {
2109         unsigned int block_nr = req->tp_block_nr;
2110         char **pg_vec;
2111         int i;
2112
2113         pg_vec = kzalloc(block_nr * sizeof(char *), GFP_KERNEL);
2114         if (unlikely(!pg_vec))
2115                 goto out;
2116
2117         for (i = 0; i < block_nr; i++) {
2118                 pg_vec[i] = alloc_one_pg_vec_page(order);
2119                 if (unlikely(!pg_vec[i]))
2120                         goto out_free_pgvec;
2121         }
2122
2123 out:
2124         return pg_vec;
2125
2126 out_free_pgvec:
2127         free_pg_vec(pg_vec, order, block_nr);
2128         pg_vec = NULL;
2129         goto out;
2130 }
2131
2132 static int packet_set_ring(struct sock *sk, struct tpacket_req *req,
2133                 int closing, int tx_ring)
2134 {
2135         char **pg_vec = NULL;
2136         struct packet_sock *po = pkt_sk(sk);
2137         int was_running, order = 0;
2138         struct packet_ring_buffer *rb;
2139         struct sk_buff_head *rb_queue;
2140         __be16 num;
2141         int err;
2142
2143         rb = tx_ring ? &po->tx_ring : &po->rx_ring;
2144         rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
2145
2146         err = -EBUSY;
2147         if (!closing) {
2148                 if (atomic_read(&po->mapped))
2149                         goto out;
2150                 if (atomic_read(&rb->pending))
2151                         goto out;
2152         }
2153
2154         if (req->tp_block_nr) {
2155                 /* Sanity tests and some calculations */
2156                 err = -EBUSY;
2157                 if (unlikely(rb->pg_vec))
2158                         goto out;
2159
2160                 switch (po->tp_version) {
2161                 case TPACKET_V1:
2162                         po->tp_hdrlen = TPACKET_HDRLEN;
2163                         break;
2164                 case TPACKET_V2:
2165                         po->tp_hdrlen = TPACKET2_HDRLEN;
2166                         break;
2167                 }
2168
2169                 err = -EINVAL;
2170                 if (unlikely((int)req->tp_block_size <= 0))
2171                         goto out;
2172                 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
2173                         goto out;
2174                 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
2175                                         po->tp_reserve))
2176                         goto out;
2177                 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
2178                         goto out;
2179
2180                 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
2181                 if (unlikely(rb->frames_per_block <= 0))
2182                         goto out;
2183                 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
2184                                         req->tp_frame_nr))
2185                         goto out;
2186
2187                 err = -ENOMEM;
2188                 order = get_order(req->tp_block_size);
2189                 pg_vec = alloc_pg_vec(req, order);
2190                 if (unlikely(!pg_vec))
2191                         goto out;
2192         }
2193         /* Done */
2194         else {
2195                 err = -EINVAL;
2196                 if (unlikely(req->tp_frame_nr))
2197                         goto out;
2198         }
2199
2200         lock_sock(sk);
2201
2202         /* Detach socket from network */
2203         spin_lock(&po->bind_lock);
2204         was_running = po->running;
2205         num = po->num;
2206         if (was_running) {
2207                 __dev_remove_pack(&po->prot_hook);
2208                 po->num = 0;
2209                 po->running = 0;
2210                 __sock_put(sk);
2211         }
2212         spin_unlock(&po->bind_lock);
2213
2214         synchronize_net();
2215
2216         err = -EBUSY;
2217         mutex_lock(&po->pg_vec_lock);
2218         if (closing || atomic_read(&po->mapped) == 0) {
2219                 err = 0;
2220 #define XC(a, b) ({ __typeof__ ((a)) __t; __t = (a); (a) = (b); __t; })
2221                 spin_lock_bh(&rb_queue->lock);
2222                 pg_vec = XC(rb->pg_vec, pg_vec);
2223                 rb->frame_max = (req->tp_frame_nr - 1);
2224                 rb->head = 0;
2225                 rb->frame_size = req->tp_frame_size;
2226                 spin_unlock_bh(&rb_queue->lock);
2227
2228                 order = XC(rb->pg_vec_order, order);
2229                 req->tp_block_nr = XC(rb->pg_vec_len, req->tp_block_nr);
2230
2231                 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
2232                 po->prot_hook.func = (po->rx_ring.pg_vec) ?
2233                                                 tpacket_rcv : packet_rcv;
2234                 skb_queue_purge(rb_queue);
2235 #undef XC
2236                 if (atomic_read(&po->mapped))
2237                         pr_err("packet_mmap: vma is busy: %d\n",
2238                                atomic_read(&po->mapped));
2239         }
2240         mutex_unlock(&po->pg_vec_lock);
2241
2242         spin_lock(&po->bind_lock);
2243         if (was_running && !po->running) {
2244                 sock_hold(sk);
2245                 po->running = 1;
2246                 po->num = num;
2247                 dev_add_pack(&po->prot_hook);
2248         }
2249         spin_unlock(&po->bind_lock);
2250
2251         release_sock(sk);
2252
2253         if (pg_vec)
2254                 free_pg_vec(pg_vec, order, req->tp_block_nr);
2255 out:
2256         return err;
2257 }
2258
2259 static int packet_mmap(struct file *file, struct socket *sock,
2260                 struct vm_area_struct *vma)
2261 {
2262         struct sock *sk = sock->sk;
2263         struct packet_sock *po = pkt_sk(sk);
2264         unsigned long size, expected_size;
2265         struct packet_ring_buffer *rb;
2266         unsigned long start;
2267         int err = -EINVAL;
2268         int i;
2269
2270         if (vma->vm_pgoff)
2271                 return -EINVAL;
2272
2273         mutex_lock(&po->pg_vec_lock);
2274
2275         expected_size = 0;
2276         for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
2277                 if (rb->pg_vec) {
2278                         expected_size += rb->pg_vec_len
2279                                                 * rb->pg_vec_pages
2280                                                 * PAGE_SIZE;
2281                 }
2282         }
2283
2284         if (expected_size == 0)
2285                 goto out;
2286
2287         size = vma->vm_end - vma->vm_start;
2288         if (size != expected_size)
2289                 goto out;
2290
2291         start = vma->vm_start;
2292         for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
2293                 if (rb->pg_vec == NULL)
2294                         continue;
2295
2296                 for (i = 0; i < rb->pg_vec_len; i++) {
2297                         struct page *page = virt_to_page(rb->pg_vec[i]);
2298                         int pg_num;
2299
2300                         for (pg_num = 0; pg_num < rb->pg_vec_pages;
2301                                         pg_num++, page++) {
2302                                 err = vm_insert_page(vma, start, page);
2303                                 if (unlikely(err))
2304                                         goto out;
2305                                 start += PAGE_SIZE;
2306                         }
2307                 }
2308         }
2309
2310         atomic_inc(&po->mapped);
2311         vma->vm_ops = &packet_mmap_ops;
2312         err = 0;
2313
2314 out:
2315         mutex_unlock(&po->pg_vec_lock);
2316         return err;
2317 }
2318 #endif
2319
2320
2321 static const struct proto_ops packet_ops_spkt = {
2322         .family =       PF_PACKET,
2323         .owner =        THIS_MODULE,
2324         .release =      packet_release,
2325         .bind =         packet_bind_spkt,
2326         .connect =      sock_no_connect,
2327         .socketpair =   sock_no_socketpair,
2328         .accept =       sock_no_accept,
2329         .getname =      packet_getname_spkt,
2330         .poll =         datagram_poll,
2331         .ioctl =        packet_ioctl,
2332         .listen =       sock_no_listen,
2333         .shutdown =     sock_no_shutdown,
2334         .setsockopt =   sock_no_setsockopt,
2335         .getsockopt =   sock_no_getsockopt,
2336         .sendmsg =      packet_sendmsg_spkt,
2337         .recvmsg =      packet_recvmsg,
2338         .mmap =         sock_no_mmap,
2339         .sendpage =     sock_no_sendpage,
2340 };
2341
2342 static const struct proto_ops packet_ops = {
2343         .family =       PF_PACKET,
2344         .owner =        THIS_MODULE,
2345         .release =      packet_release,
2346         .bind =         packet_bind,
2347         .connect =      sock_no_connect,
2348         .socketpair =   sock_no_socketpair,
2349         .accept =       sock_no_accept,
2350         .getname =      packet_getname,
2351         .poll =         packet_poll,
2352         .ioctl =        packet_ioctl,
2353         .listen =       sock_no_listen,
2354         .shutdown =     sock_no_shutdown,
2355         .setsockopt =   packet_setsockopt,
2356         .getsockopt =   packet_getsockopt,
2357         .sendmsg =      packet_sendmsg,
2358         .recvmsg =      packet_recvmsg,
2359         .mmap =         packet_mmap,
2360         .sendpage =     sock_no_sendpage,
2361 };
2362
2363 static struct net_proto_family packet_family_ops = {
2364         .family =       PF_PACKET,
2365         .create =       packet_create,
2366         .owner  =       THIS_MODULE,
2367 };
2368
2369 static struct notifier_block packet_netdev_notifier = {
2370         .notifier_call =        packet_notifier,
2371 };
2372
2373 #ifdef CONFIG_PROC_FS
2374 static inline struct sock *packet_seq_idx(struct net *net, loff_t off)
2375 {
2376         struct sock *s;
2377         struct hlist_node *node;
2378
2379         sk_for_each(s, node, &net->packet.sklist) {
2380                 if (!off--)
2381                         return s;
2382         }
2383         return NULL;
2384 }
2385
2386 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
2387         __acquires(seq_file_net(seq)->packet.sklist_lock)
2388 {
2389         struct net *net = seq_file_net(seq);
2390         read_lock(&net->packet.sklist_lock);
2391         return *pos ? packet_seq_idx(net, *pos - 1) : SEQ_START_TOKEN;
2392 }
2393
2394 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2395 {
2396         struct net *net = seq_file_net(seq);
2397         ++*pos;
2398         return  (v == SEQ_START_TOKEN)
2399                 ? sk_head(&net->packet.sklist)
2400                 : sk_next((struct sock *)v) ;
2401 }
2402
2403 static void packet_seq_stop(struct seq_file *seq, void *v)
2404         __releases(seq_file_net(seq)->packet.sklist_lock)
2405 {
2406         struct net *net = seq_file_net(seq);
2407         read_unlock(&net->packet.sklist_lock);
2408 }
2409
2410 static int packet_seq_show(struct seq_file *seq, void *v)
2411 {
2412         if (v == SEQ_START_TOKEN)
2413                 seq_puts(seq, "sk       RefCnt Type Proto  Iface R Rmem   User   Inode\n");
2414         else {
2415                 struct sock *s = v;
2416                 const struct packet_sock *po = pkt_sk(s);
2417
2418                 seq_printf(seq,
2419                            "%p %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6lu\n",
2420                            s,
2421                            atomic_read(&s->sk_refcnt),
2422                            s->sk_type,
2423                            ntohs(po->num),
2424                            po->ifindex,
2425                            po->running,
2426                            atomic_read(&s->sk_rmem_alloc),
2427                            sock_i_uid(s),
2428                            sock_i_ino(s));
2429         }
2430
2431         return 0;
2432 }
2433
2434 static const struct seq_operations packet_seq_ops = {
2435         .start  = packet_seq_start,
2436         .next   = packet_seq_next,
2437         .stop   = packet_seq_stop,
2438         .show   = packet_seq_show,
2439 };
2440
2441 static int packet_seq_open(struct inode *inode, struct file *file)
2442 {
2443         return seq_open_net(inode, file, &packet_seq_ops,
2444                             sizeof(struct seq_net_private));
2445 }
2446
2447 static const struct file_operations packet_seq_fops = {
2448         .owner          = THIS_MODULE,
2449         .open           = packet_seq_open,
2450         .read           = seq_read,
2451         .llseek         = seq_lseek,
2452         .release        = seq_release_net,
2453 };
2454
2455 #endif
2456
2457 static int packet_net_init(struct net *net)
2458 {
2459         rwlock_init(&net->packet.sklist_lock);
2460         INIT_HLIST_HEAD(&net->packet.sklist);
2461
2462         if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops))
2463                 return -ENOMEM;
2464
2465         return 0;
2466 }
2467
2468 static void packet_net_exit(struct net *net)
2469 {
2470         proc_net_remove(net, "packet");
2471 }
2472
2473 static struct pernet_operations packet_net_ops = {
2474         .init = packet_net_init,
2475         .exit = packet_net_exit,
2476 };
2477
2478
2479 static void __exit packet_exit(void)
2480 {
2481         unregister_netdevice_notifier(&packet_netdev_notifier);
2482         unregister_pernet_subsys(&packet_net_ops);
2483         sock_unregister(PF_PACKET);
2484         proto_unregister(&packet_proto);
2485 }
2486
2487 static int __init packet_init(void)
2488 {
2489         int rc = proto_register(&packet_proto, 0);
2490
2491         if (rc != 0)
2492                 goto out;
2493
2494         sock_register(&packet_family_ops);
2495         register_pernet_subsys(&packet_net_ops);
2496         register_netdevice_notifier(&packet_netdev_notifier);
2497 out:
2498         return rc;
2499 }
2500
2501 module_init(packet_init);
2502 module_exit(packet_exit);
2503 MODULE_LICENSE("GPL");
2504 MODULE_ALIAS_NETPROTO(PF_PACKET);