SAFE public projects git trees. - safe/jmp/linux-2.6/blob - net/packet/af_packet.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              PACKET - implements raw packet sockets.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *
  12  * Fixes:
  13  *              Alan Cox        :       verify_area() now used correctly
  14  *              Alan Cox        :       new skbuff lists, look ma no backlogs!
  15  *              Alan Cox        :       tidied skbuff lists.
  16  *              Alan Cox        :       Now uses generic datagram routines I
  17  *                                      added. Also fixed the peek/read crash
  18  *                                      from all old Linux datagram code.
  19  *              Alan Cox        :       Uses the improved datagram code.
  20  *              Alan Cox        :       Added NULL's for socket options.
  21  *              Alan Cox        :       Re-commented the code.
  22  *              Alan Cox        :       Use new kernel side addressing
  23  *              Rob Janssen     :       Correct MTU usage.
  24  *              Dave Platt      :       Counter leaks caused by incorrect
  25  *                                      interrupt locking and some slightly
  26  *                                      dubious gcc output. Can you read
  27  *                                      compiler: it said _VOLATILE_
  28  *      Richard Kooijman        :       Timestamp fixes.
  29  *              Alan Cox        :       New buffers. Use sk->mac.raw.
  30  *              Alan Cox        :       sendmsg/recvmsg support.
  31  *              Alan Cox        :       Protocol setting support
  32  *      Alexey Kuznetsov        :       Untied from IPv4 stack.
  33  *      Cyrus Durgin            :       Fixed kerneld for kmod.
  34  *      Michal Ostrowski        :       Module initialization cleanup.
  35  *         Ulises Alonso        :       Frame number limit removal and
  36  *                                      packet_set_ring memory leak.
  37  *              Eric Biederman  :       Allow for > 8 byte hardware addresses.
  38  *                                      The convention is that longer addresses
  39  *                                      will simply extend the hardware address
  40  *                                      byte arrays at the end of sockaddr_ll
  41  *                                      and packet_mreq.
  42  *              Johann Baudy    :       Added TX RING.
  43  *
  44  *              This program is free software; you can redistribute it and/or
  45  *              modify it under the terms of the GNU General Public License
  46  *              as published by the Free Software Foundation; either version
  47  *              2 of the License, or (at your option) any later version.
  48  *
  49  */
  50
  51 #include <linux/types.h>
  52 #include <linux/mm.h>
  53 #include <linux/capability.h>
  54 #include <linux/fcntl.h>
  55 #include <linux/socket.h>
  56 #include <linux/in.h>
  57 #include <linux/inet.h>
  58 #include <linux/netdevice.h>
  59 #include <linux/if_packet.h>
  60 #include <linux/wireless.h>
  61 #include <linux/kernel.h>
  62 #include <linux/kmod.h>
  63 #include <net/net_namespace.h>
  64 #include <net/ip.h>
  65 #include <net/protocol.h>
  66 #include <linux/skbuff.h>
  67 #include <net/sock.h>
  68 #include <linux/errno.h>
  69 #include <linux/timer.h>
  70 #include <asm/system.h>
  71 #include <asm/uaccess.h>
  72 #include <asm/ioctls.h>
  73 #include <asm/page.h>
  74 #include <asm/cacheflush.h>
  75 #include <asm/io.h>
  76 #include <linux/proc_fs.h>
  77 #include <linux/seq_file.h>
  78 #include <linux/poll.h>
  79 #include <linux/module.h>
  80 #include <linux/init.h>
  81 #include <linux/mutex.h>
  82
  83 #ifdef CONFIG_INET
  84 #include <net/inet_common.h>
  85 #endif
  86
  87 /*
  88    Assumptions:
  89    - if device has no dev->hard_header routine, it adds and removes ll header
  90      inside itself. In this case ll header is invisible outside of device,
  91      but higher levels still should reserve dev->hard_header_len.
  92      Some devices are enough clever to reallocate skb, when header
  93      will not fit to reserved space (tunnel), another ones are silly
  94      (PPP).
  95    - packet socket receives packets with pulled ll header,
  96      so that SOCK_RAW should push it back.
  97
  98 On receive:
  99 -----------
 100
 101 Incoming, dev->hard_header!=NULL
 102    mac_header -> ll header
 103    data       -> data
 104
 105 Outgoing, dev->hard_header!=NULL
 106    mac_header -> ll header
 107    data       -> ll header
 108
 109 Incoming, dev->hard_header==NULL
 110    mac_header -> UNKNOWN position. It is very likely, that it points to ll
 111                  header.  PPP makes it, that is wrong, because introduce
 112                  assymetry between rx and tx paths.
 113    data       -> data
 114
 115 Outgoing, dev->hard_header==NULL
 116    mac_header -> data. ll header is still not built!
 117    data       -> data
 118
 119 Resume
 120   If dev->hard_header==NULL we are unlikely to restore sensible ll header.
 121
 122
 123 On transmit:
 124 ------------
 125
 126 dev->hard_header != NULL
 127    mac_header -> ll header
 128    data       -> ll header
 129
 130 dev->hard_header == NULL (ll header is added by device, we cannot control it)
 131    mac_header -> data
 132    data       -> data
 133
 134    We should set nh.raw on output to correct posistion,
 135    packet classifier depends on it.
 136  */
 137
 138 /* Private packet socket structures. */
 139
 140 struct packet_mclist {
 141         struct packet_mclist    *next;
 142         int                     ifindex;
 143         int                     count;
 144         unsigned short          type;
 145         unsigned short          alen;
 146         unsigned char           addr[MAX_ADDR_LEN];
 147 };
 148 /* identical to struct packet_mreq except it has
 149  * a longer address field.
 150  */
 151 struct packet_mreq_max {
 152         int             mr_ifindex;
 153         unsigned short  mr_type;
 154         unsigned short  mr_alen;
 155         unsigned char   mr_address[MAX_ADDR_LEN];
 156 };
 157
 158 #ifdef CONFIG_PACKET_MMAP
 159 static int packet_set_ring(struct sock *sk, struct tpacket_req *req,
 160                 int closing, int tx_ring);
 161
 162 struct packet_ring_buffer {
 163         char                    **pg_vec;
 164         unsigned int            head;
 165         unsigned int            frames_per_block;
 166         unsigned int            frame_size;
 167         unsigned int            frame_max;
 168
 169         unsigned int            pg_vec_order;
 170         unsigned int            pg_vec_pages;
 171         unsigned int            pg_vec_len;
 172
 173         atomic_t                pending;
 174 };
 175
 176 struct packet_sock;
 177 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
 178 #endif
 179
 180 static void packet_flush_mclist(struct sock *sk);
 181
 182 struct packet_sock {
 183         /* struct sock has to be the first member of packet_sock */
 184         struct sock             sk;
 185         struct tpacket_stats    stats;
 186 #ifdef CONFIG_PACKET_MMAP
 187         struct packet_ring_buffer       rx_ring;
 188         struct packet_ring_buffer       tx_ring;
 189         int                     copy_thresh;
 190 #endif
 191         struct packet_type      prot_hook;
 192         spinlock_t              bind_lock;
 193         struct mutex            pg_vec_lock;
 194         unsigned int            running:1,      /* prot_hook is attached*/
 195                                 auxdata:1,
 196                                 origdev:1;
 197         int                     ifindex;        /* bound device         */
 198         __be16                  num;
 199         struct packet_mclist    *mclist;
 200 #ifdef CONFIG_PACKET_MMAP
 201         atomic_t                mapped;
 202         enum tpacket_versions   tp_version;
 203         unsigned int            tp_hdrlen;
 204         unsigned int            tp_reserve;
 205         unsigned int            tp_loss:1;
 206 #endif
 207 };
 208
 209 struct packet_skb_cb {
 210         unsigned int origlen;
 211         union {
 212                 struct sockaddr_pkt pkt;
 213                 struct sockaddr_ll ll;
 214         } sa;
 215 };
 216
 217 #define PACKET_SKB_CB(__skb)    ((struct packet_skb_cb *)((__skb)->cb))
 218
 219 #ifdef CONFIG_PACKET_MMAP
 220
 221 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
 222 {
 223         union {
 224                 struct tpacket_hdr *h1;
 225                 struct tpacket2_hdr *h2;
 226                 void *raw;
 227         } h;
 228
 229         h.raw = frame;
 230         switch (po->tp_version) {
 231         case TPACKET_V1:
 232                 h.h1->tp_status = status;
 233                 flush_dcache_page(virt_to_page(&h.h1->tp_status));
 234                 break;
 235         case TPACKET_V2:
 236                 h.h2->tp_status = status;
 237                 flush_dcache_page(virt_to_page(&h.h2->tp_status));
 238                 break;
 239         default:
 240                 pr_err("TPACKET version not supported\n");
 241                 BUG();
 242         }
 243
 244         smp_wmb();
 245 }
 246
 247 static int __packet_get_status(struct packet_sock *po, void *frame)
 248 {
 249         union {
 250                 struct tpacket_hdr *h1;
 251                 struct tpacket2_hdr *h2;
 252                 void *raw;
 253         } h;
 254
 255         smp_rmb();
 256
 257         h.raw = frame;
 258         switch (po->tp_version) {
 259         case TPACKET_V1:
 260                 flush_dcache_page(virt_to_page(&h.h1->tp_status));
 261                 return h.h1->tp_status;
 262         case TPACKET_V2:
 263                 flush_dcache_page(virt_to_page(&h.h2->tp_status));
 264                 return h.h2->tp_status;
 265         default:
 266                 pr_err("TPACKET version not supported\n");
 267                 BUG();
 268                 return 0;
 269         }
 270 }
 271
 272 static void *packet_lookup_frame(struct packet_sock *po,
 273                 struct packet_ring_buffer *rb,
 274                 unsigned int position,
 275                 int status)
 276 {
 277         unsigned int pg_vec_pos, frame_offset;
 278         union {
 279                 struct tpacket_hdr *h1;
 280                 struct tpacket2_hdr *h2;
 281                 void *raw;
 282         } h;
 283
 284         pg_vec_pos = position / rb->frames_per_block;
 285         frame_offset = position % rb->frames_per_block;
 286
 287         h.raw = rb->pg_vec[pg_vec_pos] + (frame_offset * rb->frame_size);
 288
 289         if (status != __packet_get_status(po, h.raw))
 290                 return NULL;
 291
 292         return h.raw;
 293 }
 294
 295 static inline void *packet_current_frame(struct packet_sock *po,
 296                 struct packet_ring_buffer *rb,
 297                 int status)
 298 {
 299         return packet_lookup_frame(po, rb, rb->head, status);
 300 }
 301
 302 static inline void *packet_previous_frame(struct packet_sock *po,
 303                 struct packet_ring_buffer *rb,
 304                 int status)
 305 {
 306         unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
 307         return packet_lookup_frame(po, rb, previous, status);
 308 }
 309
 310 static inline void packet_increment_head(struct packet_ring_buffer *buff)
 311 {
 312         buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
 313 }
 314
 315 #endif
 316
 317 static inline struct packet_sock *pkt_sk(struct sock *sk)
 318 {
 319         return (struct packet_sock *)sk;
 320 }
 321
 322 static void packet_sock_destruct(struct sock *sk)
 323 {
 324         WARN_ON(atomic_read(&sk->sk_rmem_alloc));
 325         WARN_ON(atomic_read(&sk->sk_wmem_alloc));
 326
 327         if (!sock_flag(sk, SOCK_DEAD)) {
 328                 pr_err("Attempt to release alive packet socket: %p\n", sk);
 329                 return;
 330         }
 331
 332         sk_refcnt_debug_dec(sk);
 333 }
 334
 335
 336 static const struct proto_ops packet_ops;
 337
 338 static const struct proto_ops packet_ops_spkt;
 339
 340 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
 341                            struct packet_type *pt, struct net_device *orig_dev)
 342 {
 343         struct sock *sk;
 344         struct sockaddr_pkt *spkt;
 345
 346         /*
 347          *      When we registered the protocol we saved the socket in the data
 348          *      field for just this event.
 349          */
 350
 351         sk = pt->af_packet_priv;
 352
 353         /*
 354          *      Yank back the headers [hope the device set this
 355          *      right or kerboom...]
 356          *
 357          *      Incoming packets have ll header pulled,
 358          *      push it back.
 359          *
 360          *      For outgoing ones skb->data == skb_mac_header(skb)
 361          *      so that this procedure is noop.
 362          */
 363
 364         if (skb->pkt_type == PACKET_LOOPBACK)
 365                 goto out;
 366
 367         if (dev_net(dev) != sock_net(sk))
 368                 goto out;
 369
 370         skb = skb_share_check(skb, GFP_ATOMIC);
 371         if (skb == NULL)
 372                 goto oom;
 373
 374         /* drop any routing info */
 375         skb_dst_drop(skb);
 376
 377         /* drop conntrack reference */
 378         nf_reset(skb);
 379
 380         spkt = &PACKET_SKB_CB(skb)->sa.pkt;
 381
 382         skb_push(skb, skb->data - skb_mac_header(skb));
 383
 384         /*
 385          *      The SOCK_PACKET socket receives _all_ frames.
 386          */
 387
 388         spkt->spkt_family = dev->type;
 389         strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
 390         spkt->spkt_protocol = skb->protocol;
 391
 392         /*
 393          *      Charge the memory to the socket. This is done specifically
 394          *      to prevent sockets using all the memory up.
 395          */
 396
 397         if (sock_queue_rcv_skb(sk, skb) == 0)
 398                 return 0;
 399
 400 out:
 401         kfree_skb(skb);
 402 oom:
 403         return 0;
 404 }
 405
 406
 407 /*
 408  *      Output a raw packet to a device layer. This bypasses all the other
 409  *      protocol layers and you must therefore supply it with a complete frame
 410  */
 411
 412 static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
 413                                struct msghdr *msg, size_t len)
 414 {
 415         struct sock *sk = sock->sk;
 416         struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
 417         struct sk_buff *skb;
 418         struct net_device *dev;
 419         __be16 proto = 0;
 420         int err;
 421
 422         /*
 423          *      Get and verify the address.
 424          */
 425
 426         if (saddr) {
 427                 if (msg->msg_namelen < sizeof(struct sockaddr))
 428                         return -EINVAL;
 429                 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
 430                         proto = saddr->spkt_protocol;
 431         } else
 432                 return -ENOTCONN;       /* SOCK_PACKET must be sent giving an address */
 433
 434         /*
 435          *      Find the device first to size check it
 436          */
 437
 438         saddr->spkt_device[13] = 0;
 439         dev = dev_get_by_name(sock_net(sk), saddr->spkt_device);
 440         err = -ENODEV;
 441         if (dev == NULL)
 442                 goto out_unlock;
 443
 444         err = -ENETDOWN;
 445         if (!(dev->flags & IFF_UP))
 446                 goto out_unlock;
 447
 448         /*
 449          * You may not queue a frame bigger than the mtu. This is the lowest level
 450          * raw protocol and you must do your own fragmentation at this level.
 451          */
 452
 453         err = -EMSGSIZE;
 454         if (len > dev->mtu + dev->hard_header_len)
 455                 goto out_unlock;
 456
 457         err = -ENOBUFS;
 458         skb = sock_wmalloc(sk, len + LL_RESERVED_SPACE(dev), 0, GFP_KERNEL);
 459
 460         /*
 461          * If the write buffer is full, then tough. At this level the user
 462          * gets to deal with the problem - do your own algorithmic backoffs.
 463          * That's far more flexible.
 464          */
 465
 466         if (skb == NULL)
 467                 goto out_unlock;
 468
 469         /*
 470          *      Fill it in
 471          */
 472
 473         /* FIXME: Save some space for broken drivers that write a
 474          * hard header at transmission time by themselves. PPP is the
 475          * notable one here. This should really be fixed at the driver level.
 476          */
 477         skb_reserve(skb, LL_RESERVED_SPACE(dev));
 478         skb_reset_network_header(skb);
 479
 480         /* Try to align data part correctly */
 481         if (dev->header_ops) {
 482                 skb->data -= dev->hard_header_len;
 483                 skb->tail -= dev->hard_header_len;
 484                 if (len < dev->hard_header_len)
 485                         skb_reset_network_header(skb);
 486         }
 487
 488         /* Returns -EFAULT on error */
 489         err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
 490         skb->protocol = proto;
 491         skb->dev = dev;
 492         skb->priority = sk->sk_priority;
 493         skb->mark = sk->sk_mark;
 494         if (err)
 495                 goto out_free;
 496
 497         /*
 498          *      Now send it
 499          */
 500
 501         dev_queue_xmit(skb);
 502         dev_put(dev);
 503         return len;
 504
 505 out_free:
 506         kfree_skb(skb);
 507 out_unlock:
 508         if (dev)
 509                 dev_put(dev);
 510         return err;
 511 }
 512
 513 static inline unsigned int run_filter(struct sk_buff *skb, struct sock *sk,
 514                                       unsigned int res)
 515 {
 516         struct sk_filter *filter;
 517
 518         rcu_read_lock_bh();
 519         filter = rcu_dereference(sk->sk_filter);
 520         if (filter != NULL)
 521                 res = sk_run_filter(skb, filter->insns, filter->len);
 522         rcu_read_unlock_bh();
 523
 524         return res;
 525 }
 526
 527 /*
 528    This function makes lazy skb cloning in hope that most of packets
 529    are discarded by BPF.
 530
 531    Note tricky part: we DO mangle shared skb! skb->data, skb->len
 532    and skb->cb are mangled. It works because (and until) packets
 533    falling here are owned by current CPU. Output packets are cloned
 534    by dev_queue_xmit_nit(), input packets are processed by net_bh
 535    sequencially, so that if we return skb to original state on exit,
 536    we will not harm anyone.
 537  */
 538
 539 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
 540                       struct packet_type *pt, struct net_device *orig_dev)
 541 {
 542         struct sock *sk;
 543         struct sockaddr_ll *sll;
 544         struct packet_sock *po;
 545         u8 *skb_head = skb->data;
 546         int skb_len = skb->len;
 547         unsigned int snaplen, res;
 548
 549         if (skb->pkt_type == PACKET_LOOPBACK)
 550                 goto drop;
 551
 552         sk = pt->af_packet_priv;
 553         po = pkt_sk(sk);
 554
 555         if (dev_net(dev) != sock_net(sk))
 556                 goto drop;
 557
 558         skb->dev = dev;
 559
 560         if (dev->header_ops) {
 561                 /* The device has an explicit notion of ll header,
 562                    exported to higher levels.
 563
 564                    Otherwise, the device hides datails of it frame
 565                    structure, so that corresponding packet head
 566                    never delivered to user.
 567                  */
 568                 if (sk->sk_type != SOCK_DGRAM)
 569                         skb_push(skb, skb->data - skb_mac_header(skb));
 570                 else if (skb->pkt_type == PACKET_OUTGOING) {
 571                         /* Special case: outgoing packets have ll header at head */
 572                         skb_pull(skb, skb_network_offset(skb));
 573                 }
 574         }
 575
 576         snaplen = skb->len;
 577
 578         res = run_filter(skb, sk, snaplen);
 579         if (!res)
 580                 goto drop_n_restore;
 581         if (snaplen > res)
 582                 snaplen = res;
 583
 584         if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
 585             (unsigned)sk->sk_rcvbuf)
 586                 goto drop_n_acct;
 587
 588         if (skb_shared(skb)) {
 589                 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
 590                 if (nskb == NULL)
 591                         goto drop_n_acct;
 592
 593                 if (skb_head != skb->data) {
 594                         skb->data = skb_head;
 595                         skb->len = skb_len;
 596                 }
 597                 kfree_skb(skb);
 598                 skb = nskb;
 599         }
 600
 601         BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
 602                      sizeof(skb->cb));
 603
 604         sll = &PACKET_SKB_CB(skb)->sa.ll;
 605         sll->sll_family = AF_PACKET;
 606         sll->sll_hatype = dev->type;
 607         sll->sll_protocol = skb->protocol;
 608         sll->sll_pkttype = skb->pkt_type;
 609         if (unlikely(po->origdev))
 610                 sll->sll_ifindex = orig_dev->ifindex;
 611         else
 612                 sll->sll_ifindex = dev->ifindex;
 613
 614         sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
 615
 616         PACKET_SKB_CB(skb)->origlen = skb->len;
 617
 618         if (pskb_trim(skb, snaplen))
 619                 goto drop_n_acct;
 620
 621         skb_set_owner_r(skb, sk);
 622         skb->dev = NULL;
 623         skb_dst_drop(skb);
 624
 625         /* drop conntrack reference */
 626         nf_reset(skb);
 627
 628         spin_lock(&sk->sk_receive_queue.lock);
 629         po->stats.tp_packets++;
 630         skb->dropcount = atomic_read(&sk->sk_drops);
 631         __skb_queue_tail(&sk->sk_receive_queue, skb);
 632         spin_unlock(&sk->sk_receive_queue.lock);
 633         sk->sk_data_ready(sk, skb->len);
 634         return 0;
 635
 636 drop_n_acct:
 637         po->stats.tp_drops = atomic_inc_return(&sk->sk_drops);
 638
 639 drop_n_restore:
 640         if (skb_head != skb->data && skb_shared(skb)) {
 641                 skb->data = skb_head;
 642                 skb->len = skb_len;
 643         }
 644 drop:
 645         consume_skb(skb);
 646         return 0;
 647 }
 648
 649 #ifdef CONFIG_PACKET_MMAP
 650 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
 651                        struct packet_type *pt, struct net_device *orig_dev)
 652 {
 653         struct sock *sk;
 654         struct packet_sock *po;
 655         struct sockaddr_ll *sll;
 656         union {
 657                 struct tpacket_hdr *h1;
 658                 struct tpacket2_hdr *h2;
 659                 void *raw;
 660         } h;
 661         u8 *skb_head = skb->data;
 662         int skb_len = skb->len;
 663         unsigned int snaplen, res;
 664         unsigned long status = TP_STATUS_LOSING|TP_STATUS_USER;
 665         unsigned short macoff, netoff, hdrlen;
 666         struct sk_buff *copy_skb = NULL;
 667         struct timeval tv;
 668         struct timespec ts;
 669
 670         if (skb->pkt_type == PACKET_LOOPBACK)
 671                 goto drop;
 672
 673         sk = pt->af_packet_priv;
 674         po = pkt_sk(sk);
 675
 676         if (dev_net(dev) != sock_net(sk))
 677                 goto drop;
 678
 679         if (dev->header_ops) {
 680                 if (sk->sk_type != SOCK_DGRAM)
 681                         skb_push(skb, skb->data - skb_mac_header(skb));
 682                 else if (skb->pkt_type == PACKET_OUTGOING) {
 683                         /* Special case: outgoing packets have ll header at head */
 684                         skb_pull(skb, skb_network_offset(skb));
 685                 }
 686         }
 687
 688         if (skb->ip_summed == CHECKSUM_PARTIAL)
 689                 status |= TP_STATUS_CSUMNOTREADY;
 690
 691         snaplen = skb->len;
 692
 693         res = run_filter(skb, sk, snaplen);
 694         if (!res)
 695                 goto drop_n_restore;
 696         if (snaplen > res)
 697                 snaplen = res;
 698
 699         if (sk->sk_type == SOCK_DGRAM) {
 700                 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
 701                                   po->tp_reserve;
 702         } else {
 703                 unsigned maclen = skb_network_offset(skb);
 704                 netoff = TPACKET_ALIGN(po->tp_hdrlen +
 705                                        (maclen < 16 ? 16 : maclen)) +
 706                         po->tp_reserve;
 707                 macoff = netoff - maclen;
 708         }
 709
 710         if (macoff + snaplen > po->rx_ring.frame_size) {
 711                 if (po->copy_thresh &&
 712                     atomic_read(&sk->sk_rmem_alloc) + skb->truesize <
 713                     (unsigned)sk->sk_rcvbuf) {
 714                         if (skb_shared(skb)) {
 715                                 copy_skb = skb_clone(skb, GFP_ATOMIC);
 716                         } else {
 717                                 copy_skb = skb_get(skb);
 718                                 skb_head = skb->data;
 719                         }
 720                         if (copy_skb)
 721                                 skb_set_owner_r(copy_skb, sk);
 722                 }
 723                 snaplen = po->rx_ring.frame_size - macoff;
 724                 if ((int)snaplen < 0)
 725                         snaplen = 0;
 726         }
 727
 728         spin_lock(&sk->sk_receive_queue.lock);
 729         h.raw = packet_current_frame(po, &po->rx_ring, TP_STATUS_KERNEL);
 730         if (!h.raw)
 731                 goto ring_is_full;
 732         packet_increment_head(&po->rx_ring);
 733         po->stats.tp_packets++;
 734         if (copy_skb) {
 735                 status |= TP_STATUS_COPY;
 736                 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
 737         }
 738         if (!po->stats.tp_drops)
 739                 status &= ~TP_STATUS_LOSING;
 740         spin_unlock(&sk->sk_receive_queue.lock);
 741
 742         skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
 743
 744         switch (po->tp_version) {
 745         case TPACKET_V1:
 746                 h.h1->tp_len = skb->len;
 747                 h.h1->tp_snaplen = snaplen;
 748                 h.h1->tp_mac = macoff;
 749                 h.h1->tp_net = netoff;
 750                 if (skb->tstamp.tv64)
 751                         tv = ktime_to_timeval(skb->tstamp);
 752                 else
 753                         do_gettimeofday(&tv);
 754                 h.h1->tp_sec = tv.tv_sec;
 755                 h.h1->tp_usec = tv.tv_usec;
 756                 hdrlen = sizeof(*h.h1);
 757                 break;
 758         case TPACKET_V2:
 759                 h.h2->tp_len = skb->len;
 760                 h.h2->tp_snaplen = snaplen;
 761                 h.h2->tp_mac = macoff;
 762                 h.h2->tp_net = netoff;
 763                 if (skb->tstamp.tv64)
 764                         ts = ktime_to_timespec(skb->tstamp);
 765                 else
 766                         getnstimeofday(&ts);
 767                 h.h2->tp_sec = ts.tv_sec;
 768                 h.h2->tp_nsec = ts.tv_nsec;
 769                 h.h2->tp_vlan_tci = skb->vlan_tci;
 770                 hdrlen = sizeof(*h.h2);
 771                 break;
 772         default:
 773                 BUG();
 774         }
 775
 776         sll = h.raw + TPACKET_ALIGN(hdrlen);
 777         sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
 778         sll->sll_family = AF_PACKET;
 779         sll->sll_hatype = dev->type;
 780         sll->sll_protocol = skb->protocol;
 781         sll->sll_pkttype = skb->pkt_type;
 782         if (unlikely(po->origdev))
 783                 sll->sll_ifindex = orig_dev->ifindex;
 784         else
 785                 sll->sll_ifindex = dev->ifindex;
 786
 787         __packet_set_status(po, h.raw, status);
 788         smp_mb();
 789         {
 790                 struct page *p_start, *p_end;
 791                 u8 *h_end = h.raw + macoff + snaplen - 1;
 792
 793                 p_start = virt_to_page(h.raw);
 794                 p_end = virt_to_page(h_end);
 795                 while (p_start <= p_end) {
 796                         flush_dcache_page(p_start);
 797                         p_start++;
 798                 }
 799         }
 800
 801         sk->sk_data_ready(sk, 0);
 802
 803 drop_n_restore:
 804         if (skb_head != skb->data && skb_shared(skb)) {
 805                 skb->data = skb_head;
 806                 skb->len = skb_len;
 807         }
 808 drop:
 809         kfree_skb(skb);
 810         return 0;
 811
 812 ring_is_full:
 813         po->stats.tp_drops++;
 814         spin_unlock(&sk->sk_receive_queue.lock);
 815
 816         sk->sk_data_ready(sk, 0);
 817         kfree_skb(copy_skb);
 818         goto drop_n_restore;
 819 }
 820
 821 static void tpacket_destruct_skb(struct sk_buff *skb)
 822 {
 823         struct packet_sock *po = pkt_sk(skb->sk);
 824         void *ph;
 825
 826         BUG_ON(skb == NULL);
 827
 828         if (likely(po->tx_ring.pg_vec)) {
 829                 ph = skb_shinfo(skb)->destructor_arg;
 830                 BUG_ON(__packet_get_status(po, ph) != TP_STATUS_SENDING);
 831                 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
 832                 atomic_dec(&po->tx_ring.pending);
 833                 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
 834         }
 835
 836         sock_wfree(skb);
 837 }
 838
 839 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
 840                 void *frame, struct net_device *dev, int size_max,
 841                 __be16 proto, unsigned char *addr)
 842 {
 843         union {
 844                 struct tpacket_hdr *h1;
 845                 struct tpacket2_hdr *h2;
 846                 void *raw;
 847         } ph;
 848         int to_write, offset, len, tp_len, nr_frags, len_max;
 849         struct socket *sock = po->sk.sk_socket;
 850         struct page *page;
 851         void *data;
 852         int err;
 853
 854         ph.raw = frame;
 855
 856         skb->protocol = proto;
 857         skb->dev = dev;
 858         skb->priority = po->sk.sk_priority;
 859         skb->mark = po->sk.sk_mark;
 860         skb_shinfo(skb)->destructor_arg = ph.raw;
 861
 862         switch (po->tp_version) {
 863         case TPACKET_V2:
 864                 tp_len = ph.h2->tp_len;
 865                 break;
 866         default:
 867                 tp_len = ph.h1->tp_len;
 868                 break;
 869         }
 870         if (unlikely(tp_len > size_max)) {
 871                 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
 872                 return -EMSGSIZE;
 873         }
 874
 875         skb_reserve(skb, LL_RESERVED_SPACE(dev));
 876         skb_reset_network_header(skb);
 877
 878         data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
 879         to_write = tp_len;
 880
 881         if (sock->type == SOCK_DGRAM) {
 882                 err = dev_hard_header(skb, dev, ntohs(proto), addr,
 883                                 NULL, tp_len);
 884                 if (unlikely(err < 0))
 885                         return -EINVAL;
 886         } else if (dev->hard_header_len) {
 887                 /* net device doesn't like empty head */
 888                 if (unlikely(tp_len <= dev->hard_header_len)) {
 889                         pr_err("packet size is too short (%d < %d)\n",
 890                                tp_len, dev->hard_header_len);
 891                         return -EINVAL;
 892                 }
 893
 894                 skb_push(skb, dev->hard_header_len);
 895                 err = skb_store_bits(skb, 0, data,
 896                                 dev->hard_header_len);
 897                 if (unlikely(err))
 898                         return err;
 899
 900                 data += dev->hard_header_len;
 901                 to_write -= dev->hard_header_len;
 902         }
 903
 904         err = -EFAULT;
 905         page = virt_to_page(data);
 906         offset = offset_in_page(data);
 907         len_max = PAGE_SIZE - offset;
 908         len = ((to_write > len_max) ? len_max : to_write);
 909
 910         skb->data_len = to_write;
 911         skb->len += to_write;
 912         skb->truesize += to_write;
 913         atomic_add(to_write, &po->sk.sk_wmem_alloc);
 914
 915         while (likely(to_write)) {
 916                 nr_frags = skb_shinfo(skb)->nr_frags;
 917
 918                 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
 919                         pr_err("Packet exceed the number of skb frags(%lu)\n",
 920                                MAX_SKB_FRAGS);
 921                         return -EFAULT;
 922                 }
 923
 924                 flush_dcache_page(page);
 925                 get_page(page);
 926                 skb_fill_page_desc(skb,
 927                                 nr_frags,
 928                                 page++, offset, len);
 929                 to_write -= len;
 930                 offset = 0;
 931                 len_max = PAGE_SIZE;
 932                 len = ((to_write > len_max) ? len_max : to_write);
 933         }
 934
 935         return tp_len;
 936 }
 937
 938 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
 939 {
 940         struct socket *sock;
 941         struct sk_buff *skb;
 942         struct net_device *dev;
 943         __be16 proto;
 944         int ifindex, err, reserve = 0;
 945         void *ph;
 946         struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
 947         int tp_len, size_max;
 948         unsigned char *addr;
 949         int len_sum = 0;
 950         int status = 0;
 951
 952         sock = po->sk.sk_socket;
 953
 954         mutex_lock(&po->pg_vec_lock);
 955
 956         err = -EBUSY;
 957         if (saddr == NULL) {
 958                 ifindex = po->ifindex;
 959                 proto   = po->num;
 960                 addr    = NULL;
 961         } else {
 962                 err = -EINVAL;
 963                 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
 964                         goto out;
 965                 if (msg->msg_namelen < (saddr->sll_halen
 966                                         + offsetof(struct sockaddr_ll,
 967                                                 sll_addr)))
 968                         goto out;
 969                 ifindex = saddr->sll_ifindex;
 970                 proto   = saddr->sll_protocol;
 971                 addr    = saddr->sll_addr;
 972         }
 973
 974         dev = dev_get_by_index(sock_net(&po->sk), ifindex);
 975         err = -ENXIO;
 976         if (unlikely(dev == NULL))
 977                 goto out;
 978
 979         reserve = dev->hard_header_len;
 980
 981         err = -ENETDOWN;
 982         if (unlikely(!(dev->flags & IFF_UP)))
 983                 goto out_put;
 984
 985         size_max = po->tx_ring.frame_size
 986                 - sizeof(struct skb_shared_info)
 987                 - po->tp_hdrlen
 988                 - LL_ALLOCATED_SPACE(dev)
 989                 - sizeof(struct sockaddr_ll);
 990
 991         if (size_max > dev->mtu + reserve)
 992                 size_max = dev->mtu + reserve;
 993
 994         do {
 995                 ph = packet_current_frame(po, &po->tx_ring,
 996                                 TP_STATUS_SEND_REQUEST);
 997
 998                 if (unlikely(ph == NULL)) {
 999                         schedule();
1000                         continue;
1001                 }
1002
1003                 status = TP_STATUS_SEND_REQUEST;
1004                 skb = sock_alloc_send_skb(&po->sk,
1005                                 LL_ALLOCATED_SPACE(dev)
1006                                 + sizeof(struct sockaddr_ll),
1007                                 0, &err);
1008
1009                 if (unlikely(skb == NULL))
1010                         goto out_status;
1011
1012                 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
1013                                 addr);
1014
1015                 if (unlikely(tp_len < 0)) {
1016                         if (po->tp_loss) {
1017                                 __packet_set_status(po, ph,
1018                                                 TP_STATUS_AVAILABLE);
1019                                 packet_increment_head(&po->tx_ring);
1020                                 kfree_skb(skb);
1021                                 continue;
1022                         } else {
1023                                 status = TP_STATUS_WRONG_FORMAT;
1024                                 err = tp_len;
1025                                 goto out_status;
1026                         }
1027                 }
1028
1029                 skb->destructor = tpacket_destruct_skb;
1030                 __packet_set_status(po, ph, TP_STATUS_SENDING);
1031                 atomic_inc(&po->tx_ring.pending);
1032
1033                 status = TP_STATUS_SEND_REQUEST;
1034                 err = dev_queue_xmit(skb);
1035                 if (unlikely(err > 0 && (err = net_xmit_errno(err)) != 0))
1036                         goto out_xmit;
1037                 packet_increment_head(&po->tx_ring);
1038                 len_sum += tp_len;
1039         } while (likely((ph != NULL) || ((!(msg->msg_flags & MSG_DONTWAIT))
1040                                         && (atomic_read(&po->tx_ring.pending))))
1041               );
1042
1043         err = len_sum;
1044         goto out_put;
1045
1046 out_xmit:
1047         skb->destructor = sock_wfree;
1048         atomic_dec(&po->tx_ring.pending);
1049 out_status:
1050         __packet_set_status(po, ph, status);
1051         kfree_skb(skb);
1052 out_put:
1053         dev_put(dev);
1054 out:
1055         mutex_unlock(&po->pg_vec_lock);
1056         return err;
1057 }
1058 #endif
1059
1060 static int packet_snd(struct socket *sock,
1061                           struct msghdr *msg, size_t len)
1062 {
1063         struct sock *sk = sock->sk;
1064         struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1065         struct sk_buff *skb;
1066         struct net_device *dev;
1067         __be16 proto;
1068         unsigned char *addr;
1069         int ifindex, err, reserve = 0;
1070
1071         /*
1072          *      Get and verify the address.
1073          */
1074
1075         if (saddr == NULL) {
1076                 struct packet_sock *po = pkt_sk(sk);
1077
1078                 ifindex = po->ifindex;
1079                 proto   = po->num;
1080                 addr    = NULL;
1081         } else {
1082                 err = -EINVAL;
1083                 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
1084                         goto out;
1085                 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
1086                         goto out;
1087                 ifindex = saddr->sll_ifindex;
1088                 proto   = saddr->sll_protocol;
1089                 addr    = saddr->sll_addr;
1090         }
1091
1092
1093         dev = dev_get_by_index(sock_net(sk), ifindex);
1094         err = -ENXIO;
1095         if (dev == NULL)
1096                 goto out_unlock;
1097         if (sock->type == SOCK_RAW)
1098                 reserve = dev->hard_header_len;
1099
1100         err = -ENETDOWN;
1101         if (!(dev->flags & IFF_UP))
1102                 goto out_unlock;
1103
1104         err = -EMSGSIZE;
1105         if (len > dev->mtu+reserve)
1106                 goto out_unlock;
1107
1108         skb = sock_alloc_send_skb(sk, len + LL_ALLOCATED_SPACE(dev),
1109                                 msg->msg_flags & MSG_DONTWAIT, &err);
1110         if (skb == NULL)
1111                 goto out_unlock;
1112
1113         skb_reserve(skb, LL_RESERVED_SPACE(dev));
1114         skb_reset_network_header(skb);
1115
1116         err = -EINVAL;
1117         if (sock->type == SOCK_DGRAM &&
1118             dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len) < 0)
1119                 goto out_free;
1120
1121         /* Returns -EFAULT on error */
1122         err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1123         if (err)
1124                 goto out_free;
1125
1126         skb->protocol = proto;
1127         skb->dev = dev;
1128         skb->priority = sk->sk_priority;
1129         skb->mark = sk->sk_mark;
1130
1131         /*
1132          *      Now send it
1133          */
1134
1135         err = dev_queue_xmit(skb);
1136         if (err > 0 && (err = net_xmit_errno(err)) != 0)
1137                 goto out_unlock;
1138
1139         dev_put(dev);
1140
1141         return len;
1142
1143 out_free:
1144         kfree_skb(skb);
1145 out_unlock:
1146         if (dev)
1147                 dev_put(dev);
1148 out:
1149         return err;
1150 }
1151
1152 static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
1153                 struct msghdr *msg, size_t len)
1154 {
1155 #ifdef CONFIG_PACKET_MMAP
1156         struct sock *sk = sock->sk;
1157         struct packet_sock *po = pkt_sk(sk);
1158         if (po->tx_ring.pg_vec)
1159                 return tpacket_snd(po, msg);
1160         else
1161 #endif
1162                 return packet_snd(sock, msg, len);
1163 }
1164
1165 /*
1166  *      Close a PACKET socket. This is fairly simple. We immediately go
1167  *      to 'closed' state and remove our protocol entry in the device list.
1168  */
1169
1170 static int packet_release(struct socket *sock)
1171 {
1172         struct sock *sk = sock->sk;
1173         struct packet_sock *po;
1174         struct net *net;
1175 #ifdef CONFIG_PACKET_MMAP
1176         struct tpacket_req req;
1177 #endif
1178
1179         if (!sk)
1180                 return 0;
1181
1182         net = sock_net(sk);
1183         po = pkt_sk(sk);
1184
1185         write_lock_bh(&net->packet.sklist_lock);
1186         sk_del_node_init(sk);
1187         sock_prot_inuse_add(net, sk->sk_prot, -1);
1188         write_unlock_bh(&net->packet.sklist_lock);
1189
1190         /*
1191          *      Unhook packet receive handler.
1192          */
1193
1194         if (po->running) {
1195                 /*
1196                  *      Remove the protocol hook
1197                  */
1198                 dev_remove_pack(&po->prot_hook);
1199                 po->running = 0;
1200                 po->num = 0;
1201                 __sock_put(sk);
1202         }
1203
1204         packet_flush_mclist(sk);
1205
1206 #ifdef CONFIG_PACKET_MMAP
1207         memset(&req, 0, sizeof(req));
1208
1209         if (po->rx_ring.pg_vec)
1210                 packet_set_ring(sk, &req, 1, 0);
1211
1212         if (po->tx_ring.pg_vec)
1213                 packet_set_ring(sk, &req, 1, 1);
1214 #endif
1215
1216         /*
1217          *      Now the socket is dead. No more input will appear.
1218          */
1219
1220         sock_orphan(sk);
1221         sock->sk = NULL;
1222
1223         /* Purge queues */
1224
1225         skb_queue_purge(&sk->sk_receive_queue);
1226         sk_refcnt_debug_release(sk);
1227
1228         sock_put(sk);
1229         return 0;
1230 }
1231
1232 /*
1233  *      Attach a packet hook.
1234  */
1235
1236 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1237 {
1238         struct packet_sock *po = pkt_sk(sk);
1239         /*
1240          *      Detach an existing hook if present.
1241          */
1242
1243         lock_sock(sk);
1244
1245         spin_lock(&po->bind_lock);
1246         if (po->running) {
1247                 __sock_put(sk);
1248                 po->running = 0;
1249                 po->num = 0;
1250                 spin_unlock(&po->bind_lock);
1251                 dev_remove_pack(&po->prot_hook);
1252                 spin_lock(&po->bind_lock);
1253         }
1254
1255         po->num = protocol;
1256         po->prot_hook.type = protocol;
1257         po->prot_hook.dev = dev;
1258
1259         po->ifindex = dev ? dev->ifindex : 0;
1260
1261         if (protocol == 0)
1262                 goto out_unlock;
1263
1264         if (!dev || (dev->flags & IFF_UP)) {
1265                 dev_add_pack(&po->prot_hook);
1266                 sock_hold(sk);
1267                 po->running = 1;
1268         } else {
1269                 sk->sk_err = ENETDOWN;
1270                 if (!sock_flag(sk, SOCK_DEAD))
1271                         sk->sk_error_report(sk);
1272         }
1273
1274 out_unlock:
1275         spin_unlock(&po->bind_lock);
1276         release_sock(sk);
1277         return 0;
1278 }
1279
1280 /*
1281  *      Bind a packet socket to a device
1282  */
1283
1284 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
1285                             int addr_len)
1286 {
1287         struct sock *sk = sock->sk;
1288         char name[15];
1289         struct net_device *dev;
1290         int err = -ENODEV;
1291
1292         /*
1293          *      Check legality
1294          */
1295
1296         if (addr_len != sizeof(struct sockaddr))
1297                 return -EINVAL;
1298         strlcpy(name, uaddr->sa_data, sizeof(name));
1299
1300         dev = dev_get_by_name(sock_net(sk), name);
1301         if (dev) {
1302                 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1303                 dev_put(dev);
1304         }
1305         return err;
1306 }
1307
1308 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
1309 {
1310         struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
1311         struct sock *sk = sock->sk;
1312         struct net_device *dev = NULL;
1313         int err;
1314
1315
1316         /*
1317          *      Check legality
1318          */
1319
1320         if (addr_len < sizeof(struct sockaddr_ll))
1321                 return -EINVAL;
1322         if (sll->sll_family != AF_PACKET)
1323                 return -EINVAL;
1324
1325         if (sll->sll_ifindex) {
1326                 err = -ENODEV;
1327                 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1328                 if (dev == NULL)
1329                         goto out;
1330         }
1331         err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1332         if (dev)
1333                 dev_put(dev);
1334
1335 out:
1336         return err;
1337 }
1338
1339 static struct proto packet_proto = {
1340         .name     = "PACKET",
1341         .owner    = THIS_MODULE,
1342         .obj_size = sizeof(struct packet_sock),
1343 };
1344
1345 /*
1346  *      Create a packet of type SOCK_PACKET.
1347  */
1348
1349 static int packet_create(struct net *net, struct socket *sock, int protocol)
1350 {
1351         struct sock *sk;
1352         struct packet_sock *po;
1353         __be16 proto = (__force __be16)protocol; /* weird, but documented */
1354         int err;
1355
1356         if (!capable(CAP_NET_RAW))
1357                 return -EPERM;
1358         if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
1359             sock->type != SOCK_PACKET)
1360                 return -ESOCKTNOSUPPORT;
1361
1362         sock->state = SS_UNCONNECTED;
1363
1364         err = -ENOBUFS;
1365         sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1366         if (sk == NULL)
1367                 goto out;
1368
1369         sock->ops = &packet_ops;
1370         if (sock->type == SOCK_PACKET)
1371                 sock->ops = &packet_ops_spkt;
1372
1373         sock_init_data(sock, sk);
1374
1375         po = pkt_sk(sk);
1376         sk->sk_family = PF_PACKET;
1377         po->num = proto;
1378
1379         sk->sk_destruct = packet_sock_destruct;
1380         sk_refcnt_debug_inc(sk);
1381
1382         /*
1383          *      Attach a protocol block
1384          */
1385
1386         spin_lock_init(&po->bind_lock);
1387         mutex_init(&po->pg_vec_lock);
1388         po->prot_hook.func = packet_rcv;
1389
1390         if (sock->type == SOCK_PACKET)
1391                 po->prot_hook.func = packet_rcv_spkt;
1392
1393         po->prot_hook.af_packet_priv = sk;
1394
1395         if (proto) {
1396                 po->prot_hook.type = proto;
1397                 dev_add_pack(&po->prot_hook);
1398                 sock_hold(sk);
1399                 po->running = 1;
1400         }
1401
1402         write_lock_bh(&net->packet.sklist_lock);
1403         sk_add_node(sk, &net->packet.sklist);
1404         sock_prot_inuse_add(net, &packet_proto, 1);
1405         write_unlock_bh(&net->packet.sklist_lock);
1406         return 0;
1407 out:
1408         return err;
1409 }
1410
1411 /*
1412  *      Pull a packet from our receive queue and hand it to the user.
1413  *      If necessary we block.
1414  */
1415
1416 static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
1417                           struct msghdr *msg, size_t len, int flags)
1418 {
1419         struct sock *sk = sock->sk;
1420         struct sk_buff *skb;
1421         int copied, err;
1422         struct sockaddr_ll *sll;
1423
1424         err = -EINVAL;
1425         if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT))
1426                 goto out;
1427
1428 #if 0
1429         /* What error should we return now? EUNATTACH? */
1430         if (pkt_sk(sk)->ifindex < 0)
1431                 return -ENODEV;
1432 #endif
1433
1434         /*
1435          *      Call the generic datagram receiver. This handles all sorts
1436          *      of horrible races and re-entrancy so we can forget about it
1437          *      in the protocol layers.
1438          *
1439          *      Now it will return ENETDOWN, if device have just gone down,
1440          *      but then it will block.
1441          */
1442
1443         skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1444
1445         /*
1446          *      An error occurred so return it. Because skb_recv_datagram()
1447          *      handles the blocking we don't see and worry about blocking
1448          *      retries.
1449          */
1450
1451         if (skb == NULL)
1452                 goto out;
1453
1454         /*
1455          *      If the address length field is there to be filled in, we fill
1456          *      it in now.
1457          */
1458
1459         sll = &PACKET_SKB_CB(skb)->sa.ll;
1460         if (sock->type == SOCK_PACKET)
1461                 msg->msg_namelen = sizeof(struct sockaddr_pkt);
1462         else
1463                 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
1464
1465         /*
1466          *      You lose any data beyond the buffer you gave. If it worries a
1467          *      user program they can ask the device for its MTU anyway.
1468          */
1469
1470         copied = skb->len;
1471         if (copied > len) {
1472                 copied = len;
1473                 msg->msg_flags |= MSG_TRUNC;
1474         }
1475
1476         err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
1477         if (err)
1478                 goto out_free;
1479
1480         sock_recv_ts_and_drops(msg, sk, skb);
1481
1482         if (msg->msg_name)
1483                 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
1484                        msg->msg_namelen);
1485
1486         if (pkt_sk(sk)->auxdata) {
1487                 struct tpacket_auxdata aux;
1488
1489                 aux.tp_status = TP_STATUS_USER;
1490                 if (skb->ip_summed == CHECKSUM_PARTIAL)
1491                         aux.tp_status |= TP_STATUS_CSUMNOTREADY;
1492                 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
1493                 aux.tp_snaplen = skb->len;
1494                 aux.tp_mac = 0;
1495                 aux.tp_net = skb_network_offset(skb);
1496                 aux.tp_vlan_tci = skb->vlan_tci;
1497
1498                 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
1499         }
1500
1501         /*
1502          *      Free or return the buffer as appropriate. Again this
1503          *      hides all the races and re-entrancy issues from us.
1504          */
1505         err = (flags&MSG_TRUNC) ? skb->len : copied;
1506
1507 out_free:
1508         skb_free_datagram(sk, skb);
1509 out:
1510         return err;
1511 }
1512
1513 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
1514                                int *uaddr_len, int peer)
1515 {
1516         struct net_device *dev;
1517         struct sock *sk = sock->sk;
1518
1519         if (peer)
1520                 return -EOPNOTSUPP;
1521
1522         uaddr->sa_family = AF_PACKET;
1523         dev = dev_get_by_index(sock_net(sk), pkt_sk(sk)->ifindex);
1524         if (dev) {
1525                 strlcpy(uaddr->sa_data, dev->name, 15);
1526                 dev_put(dev);
1527         } else
1528                 memset(uaddr->sa_data, 0, 14);
1529         *uaddr_len = sizeof(*uaddr);
1530
1531         return 0;
1532 }
1533
1534 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
1535                           int *uaddr_len, int peer)
1536 {
1537         struct net_device *dev;
1538         struct sock *sk = sock->sk;
1539         struct packet_sock *po = pkt_sk(sk);
1540         struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
1541
1542         if (peer)
1543                 return -EOPNOTSUPP;
1544
1545         sll->sll_family = AF_PACKET;
1546         sll->sll_ifindex = po->ifindex;
1547         sll->sll_protocol = po->num;
1548         dev = dev_get_by_index(sock_net(sk), po->ifindex);
1549         if (dev) {
1550                 sll->sll_hatype = dev->type;
1551                 sll->sll_halen = dev->addr_len;
1552                 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1553                 dev_put(dev);
1554         } else {
1555                 sll->sll_hatype = 0;    /* Bad: we have no ARPHRD_UNSPEC */
1556                 sll->sll_halen = 0;
1557         }
1558         *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1559
1560         return 0;
1561 }
1562
1563 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
1564                          int what)
1565 {
1566         switch (i->type) {
1567         case PACKET_MR_MULTICAST:
1568                 if (what > 0)
1569                         return dev_mc_add(dev, i->addr, i->alen, 0);
1570                 else
1571                         return dev_mc_delete(dev, i->addr, i->alen, 0);
1572                 break;
1573         case PACKET_MR_PROMISC:
1574                 return dev_set_promiscuity(dev, what);
1575                 break;
1576         case PACKET_MR_ALLMULTI:
1577                 return dev_set_allmulti(dev, what);
1578                 break;
1579         case PACKET_MR_UNICAST:
1580                 if (what > 0)
1581                         return dev_unicast_add(dev, i->addr);
1582                 else
1583                         return dev_unicast_delete(dev, i->addr);
1584                 break;
1585         default:
1586                 break;
1587         }
1588         return 0;
1589 }
1590
1591 static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
1592 {
1593         for ( ; i; i = i->next) {
1594                 if (i->ifindex == dev->ifindex)
1595                         packet_dev_mc(dev, i, what);
1596         }
1597 }
1598
1599 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1600 {
1601         struct packet_sock *po = pkt_sk(sk);
1602         struct packet_mclist *ml, *i;
1603         struct net_device *dev;
1604         int err;
1605
1606         rtnl_lock();
1607
1608         err = -ENODEV;
1609         dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1610         if (!dev)
1611                 goto done;
1612
1613         err = -EINVAL;
1614         if (mreq->mr_alen > dev->addr_len)
1615                 goto done;
1616
1617         err = -ENOBUFS;
1618         i = kmalloc(sizeof(*i), GFP_KERNEL);
1619         if (i == NULL)
1620                 goto done;
1621
1622         err = 0;
1623         for (ml = po->mclist; ml; ml = ml->next) {
1624                 if (ml->ifindex == mreq->mr_ifindex &&
1625                     ml->type == mreq->mr_type &&
1626                     ml->alen == mreq->mr_alen &&
1627                     memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
1628                         ml->count++;
1629                         /* Free the new element ... */
1630                         kfree(i);
1631                         goto done;
1632                 }
1633         }
1634
1635         i->type = mreq->mr_type;
1636         i->ifindex = mreq->mr_ifindex;
1637         i->alen = mreq->mr_alen;
1638         memcpy(i->addr, mreq->mr_address, i->alen);
1639         i->count = 1;
1640         i->next = po->mclist;
1641         po->mclist = i;
1642         err = packet_dev_mc(dev, i, 1);
1643         if (err) {
1644                 po->mclist = i->next;
1645                 kfree(i);
1646         }
1647
1648 done:
1649         rtnl_unlock();
1650         return err;
1651 }
1652
1653 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1654 {
1655         struct packet_mclist *ml, **mlp;
1656
1657         rtnl_lock();
1658
1659         for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
1660                 if (ml->ifindex == mreq->mr_ifindex &&
1661                     ml->type == mreq->mr_type &&
1662                     ml->alen == mreq->mr_alen &&
1663                     memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
1664                         if (--ml->count == 0) {
1665                                 struct net_device *dev;
1666                                 *mlp = ml->next;
1667                                 dev = dev_get_by_index(sock_net(sk), ml->ifindex);
1668                                 if (dev) {
1669                                         packet_dev_mc(dev, ml, -1);
1670                                         dev_put(dev);
1671                                 }
1672                                 kfree(ml);
1673                         }
1674                         rtnl_unlock();
1675                         return 0;
1676                 }
1677         }
1678         rtnl_unlock();
1679         return -EADDRNOTAVAIL;
1680 }
1681
1682 static void packet_flush_mclist(struct sock *sk)
1683 {
1684         struct packet_sock *po = pkt_sk(sk);
1685         struct packet_mclist *ml;
1686
1687         if (!po->mclist)
1688                 return;
1689
1690         rtnl_lock();
1691         while ((ml = po->mclist) != NULL) {
1692                 struct net_device *dev;
1693
1694                 po->mclist = ml->next;
1695                 dev = dev_get_by_index(sock_net(sk), ml->ifindex);
1696                 if (dev != NULL) {
1697                         packet_dev_mc(dev, ml, -1);
1698                         dev_put(dev);
1699                 }
1700                 kfree(ml);
1701         }
1702         rtnl_unlock();
1703 }
1704
1705 static int
1706 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1707 {
1708         struct sock *sk = sock->sk;
1709         struct packet_sock *po = pkt_sk(sk);
1710         int ret;
1711
1712         if (level != SOL_PACKET)
1713                 return -ENOPROTOOPT;
1714
1715         switch (optname) {
1716         case PACKET_ADD_MEMBERSHIP:
1717         case PACKET_DROP_MEMBERSHIP:
1718         {
1719                 struct packet_mreq_max mreq;
1720                 int len = optlen;
1721                 memset(&mreq, 0, sizeof(mreq));
1722                 if (len < sizeof(struct packet_mreq))
1723                         return -EINVAL;
1724                 if (len > sizeof(mreq))
1725                         len = sizeof(mreq);
1726                 if (copy_from_user(&mreq, optval, len))
1727                         return -EFAULT;
1728                 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
1729                         return -EINVAL;
1730                 if (optname == PACKET_ADD_MEMBERSHIP)
1731                         ret = packet_mc_add(sk, &mreq);
1732                 else
1733                         ret = packet_mc_drop(sk, &mreq);
1734                 return ret;
1735         }
1736
1737 #ifdef CONFIG_PACKET_MMAP
1738         case PACKET_RX_RING:
1739         case PACKET_TX_RING:
1740         {
1741                 struct tpacket_req req;
1742
1743                 if (optlen < sizeof(req))
1744                         return -EINVAL;
1745                 if (copy_from_user(&req, optval, sizeof(req)))
1746                         return -EFAULT;
1747                 return packet_set_ring(sk, &req, 0, optname == PACKET_TX_RING);
1748         }
1749         case PACKET_COPY_THRESH:
1750         {
1751                 int val;
1752
1753                 if (optlen != sizeof(val))
1754                         return -EINVAL;
1755                 if (copy_from_user(&val, optval, sizeof(val)))
1756                         return -EFAULT;
1757
1758                 pkt_sk(sk)->copy_thresh = val;
1759                 return 0;
1760         }
1761         case PACKET_VERSION:
1762         {
1763                 int val;
1764
1765                 if (optlen != sizeof(val))
1766                         return -EINVAL;
1767                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1768                         return -EBUSY;
1769                 if (copy_from_user(&val, optval, sizeof(val)))
1770                         return -EFAULT;
1771                 switch (val) {
1772                 case TPACKET_V1:
1773                 case TPACKET_V2:
1774                         po->tp_version = val;
1775                         return 0;
1776                 default:
1777                         return -EINVAL;
1778                 }
1779         }
1780         case PACKET_RESERVE:
1781         {
1782                 unsigned int val;
1783
1784                 if (optlen != sizeof(val))
1785                         return -EINVAL;
1786                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1787                         return -EBUSY;
1788                 if (copy_from_user(&val, optval, sizeof(val)))
1789                         return -EFAULT;
1790                 po->tp_reserve = val;
1791                 return 0;
1792         }
1793         case PACKET_LOSS:
1794         {
1795                 unsigned int val;
1796
1797                 if (optlen != sizeof(val))
1798                         return -EINVAL;
1799                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1800                         return -EBUSY;
1801                 if (copy_from_user(&val, optval, sizeof(val)))
1802                         return -EFAULT;
1803                 po->tp_loss = !!val;
1804                 return 0;
1805         }
1806 #endif
1807         case PACKET_AUXDATA:
1808         {
1809                 int val;
1810
1811                 if (optlen < sizeof(val))
1812                         return -EINVAL;
1813                 if (copy_from_user(&val, optval, sizeof(val)))
1814                         return -EFAULT;
1815
1816                 po->auxdata = !!val;
1817                 return 0;
1818         }
1819         case PACKET_ORIGDEV:
1820         {
1821                 int val;
1822
1823                 if (optlen < sizeof(val))
1824                         return -EINVAL;
1825                 if (copy_from_user(&val, optval, sizeof(val)))
1826                         return -EFAULT;
1827
1828                 po->origdev = !!val;
1829                 return 0;
1830         }
1831         default:
1832                 return -ENOPROTOOPT;
1833         }
1834 }
1835
1836 static int packet_getsockopt(struct socket *sock, int level, int optname,
1837                              char __user *optval, int __user *optlen)
1838 {
1839         int len;
1840         int val;
1841         struct sock *sk = sock->sk;
1842         struct packet_sock *po = pkt_sk(sk);
1843         void *data;
1844         struct tpacket_stats st;
1845
1846         if (level != SOL_PACKET)
1847                 return -ENOPROTOOPT;
1848
1849         if (get_user(len, optlen))
1850                 return -EFAULT;
1851
1852         if (len < 0)
1853                 return -EINVAL;
1854
1855         switch (optname) {
1856         case PACKET_STATISTICS:
1857                 if (len > sizeof(struct tpacket_stats))
1858                         len = sizeof(struct tpacket_stats);
1859                 spin_lock_bh(&sk->sk_receive_queue.lock);
1860                 st = po->stats;
1861                 memset(&po->stats, 0, sizeof(st));
1862                 spin_unlock_bh(&sk->sk_receive_queue.lock);
1863                 st.tp_packets += st.tp_drops;
1864
1865                 data = &st;
1866                 break;
1867         case PACKET_AUXDATA:
1868                 if (len > sizeof(int))
1869                         len = sizeof(int);
1870                 val = po->auxdata;
1871
1872                 data = &val;
1873                 break;
1874         case PACKET_ORIGDEV:
1875                 if (len > sizeof(int))
1876                         len = sizeof(int);
1877                 val = po->origdev;
1878
1879                 data = &val;
1880                 break;
1881 #ifdef CONFIG_PACKET_MMAP
1882         case PACKET_VERSION:
1883                 if (len > sizeof(int))
1884                         len = sizeof(int);
1885                 val = po->tp_version;
1886                 data = &val;
1887                 break;
1888         case PACKET_HDRLEN:
1889                 if (len > sizeof(int))
1890                         len = sizeof(int);
1891                 if (copy_from_user(&val, optval, len))
1892                         return -EFAULT;
1893                 switch (val) {
1894                 case TPACKET_V1:
1895                         val = sizeof(struct tpacket_hdr);
1896                         break;
1897                 case TPACKET_V2:
1898                         val = sizeof(struct tpacket2_hdr);
1899                         break;
1900                 default:
1901                         return -EINVAL;
1902                 }
1903                 data = &val;
1904                 break;
1905         case PACKET_RESERVE:
1906                 if (len > sizeof(unsigned int))
1907                         len = sizeof(unsigned int);
1908                 val = po->tp_reserve;
1909                 data = &val;
1910                 break;
1911         case PACKET_LOSS:
1912                 if (len > sizeof(unsigned int))
1913                         len = sizeof(unsigned int);
1914                 val = po->tp_loss;
1915                 data = &val;
1916                 break;
1917 #endif
1918         default:
1919                 return -ENOPROTOOPT;
1920         }
1921
1922         if (put_user(len, optlen))
1923                 return -EFAULT;
1924         if (copy_to_user(optval, data, len))
1925                 return -EFAULT;
1926         return 0;
1927 }
1928
1929
1930 static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
1931 {
1932         struct sock *sk;
1933         struct hlist_node *node;
1934         struct net_device *dev = data;
1935         struct net *net = dev_net(dev);
1936
1937         read_lock(&net->packet.sklist_lock);
1938         sk_for_each(sk, node, &net->packet.sklist) {
1939                 struct packet_sock *po = pkt_sk(sk);
1940
1941                 switch (msg) {
1942                 case NETDEV_UNREGISTER:
1943                         if (po->mclist)
1944                                 packet_dev_mclist(dev, po->mclist, -1);
1945                         /* fallthrough */
1946
1947                 case NETDEV_DOWN:
1948                         if (dev->ifindex == po->ifindex) {
1949                                 spin_lock(&po->bind_lock);
1950                                 if (po->running) {
1951                                         __dev_remove_pack(&po->prot_hook);
1952                                         __sock_put(sk);
1953                                         po->running = 0;
1954                                         sk->sk_err = ENETDOWN;
1955                                         if (!sock_flag(sk, SOCK_DEAD))
1956                                                 sk->sk_error_report(sk);
1957                                 }
1958                                 if (msg == NETDEV_UNREGISTER) {
1959                                         po->ifindex = -1;
1960                                         po->prot_hook.dev = NULL;
1961                                 }
1962                                 spin_unlock(&po->bind_lock);
1963                         }
1964                         break;
1965                 case NETDEV_UP:
1966                         spin_lock(&po->bind_lock);
1967                         if (dev->ifindex == po->ifindex && po->num &&
1968                             !po->running) {
1969                                 dev_add_pack(&po->prot_hook);
1970                                 sock_hold(sk);
1971                                 po->running = 1;
1972                         }
1973                         spin_unlock(&po->bind_lock);
1974                         break;
1975                 }
1976         }
1977         read_unlock(&net->packet.sklist_lock);
1978         return NOTIFY_DONE;
1979 }
1980
1981
1982 static int packet_ioctl(struct socket *sock, unsigned int cmd,
1983                         unsigned long arg)
1984 {
1985         struct sock *sk = sock->sk;
1986
1987         switch (cmd) {
1988         case SIOCOUTQ:
1989         {
1990                 int amount = sk_wmem_alloc_get(sk);
1991
1992                 return put_user(amount, (int __user *)arg);
1993         }
1994         case SIOCINQ:
1995         {
1996                 struct sk_buff *skb;
1997                 int amount = 0;
1998
1999                 spin_lock_bh(&sk->sk_receive_queue.lock);
2000                 skb = skb_peek(&sk->sk_receive_queue);
2001                 if (skb)
2002                         amount = skb->len;
2003                 spin_unlock_bh(&sk->sk_receive_queue.lock);
2004                 return put_user(amount, (int __user *)arg);
2005         }
2006         case SIOCGSTAMP:
2007                 return sock_get_timestamp(sk, (struct timeval __user *)arg);
2008         case SIOCGSTAMPNS:
2009                 return sock_get_timestampns(sk, (struct timespec __user *)arg);
2010
2011 #ifdef CONFIG_INET
2012         case SIOCADDRT:
2013         case SIOCDELRT:
2014         case SIOCDARP:
2015         case SIOCGARP:
2016         case SIOCSARP:
2017         case SIOCGIFADDR:
2018         case SIOCSIFADDR:
2019         case SIOCGIFBRDADDR:
2020         case SIOCSIFBRDADDR:
2021         case SIOCGIFNETMASK:
2022         case SIOCSIFNETMASK:
2023         case SIOCGIFDSTADDR:
2024         case SIOCSIFDSTADDR:
2025         case SIOCSIFFLAGS:
2026                 if (!net_eq(sock_net(sk), &init_net))
2027                         return -ENOIOCTLCMD;
2028                 return inet_dgram_ops.ioctl(sock, cmd, arg);
2029 #endif
2030
2031         default:
2032                 return -ENOIOCTLCMD;
2033         }
2034         return 0;
2035 }
2036
2037 #ifndef CONFIG_PACKET_MMAP
2038 #define packet_mmap sock_no_mmap
2039 #define packet_poll datagram_poll
2040 #else
2041
2042 static unsigned int packet_poll(struct file *file, struct socket *sock,
2043                                 poll_table *wait)
2044 {
2045         struct sock *sk = sock->sk;
2046         struct packet_sock *po = pkt_sk(sk);
2047         unsigned int mask = datagram_poll(file, sock, wait);
2048
2049         spin_lock_bh(&sk->sk_receive_queue.lock);
2050         if (po->rx_ring.pg_vec) {
2051                 if (!packet_previous_frame(po, &po->rx_ring, TP_STATUS_KERNEL))
2052                         mask |= POLLIN | POLLRDNORM;
2053         }
2054         spin_unlock_bh(&sk->sk_receive_queue.lock);
2055         spin_lock_bh(&sk->sk_write_queue.lock);
2056         if (po->tx_ring.pg_vec) {
2057                 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
2058                         mask |= POLLOUT | POLLWRNORM;
2059         }
2060         spin_unlock_bh(&sk->sk_write_queue.lock);
2061         return mask;
2062 }
2063
2064
2065 /* Dirty? Well, I still did not learn better way to account
2066  * for user mmaps.
2067  */
2068
2069 static void packet_mm_open(struct vm_area_struct *vma)
2070 {
2071         struct file *file = vma->vm_file;
2072         struct socket *sock = file->private_data;
2073         struct sock *sk = sock->sk;
2074
2075         if (sk)
2076                 atomic_inc(&pkt_sk(sk)->mapped);
2077 }
2078
2079 static void packet_mm_close(struct vm_area_struct *vma)
2080 {
2081         struct file *file = vma->vm_file;
2082         struct socket *sock = file->private_data;
2083         struct sock *sk = sock->sk;
2084
2085         if (sk)
2086                 atomic_dec(&pkt_sk(sk)->mapped);
2087 }
2088
2089 static const struct vm_operations_struct packet_mmap_ops = {
2090         .open   =       packet_mm_open,
2091         .close  =       packet_mm_close,
2092 };
2093
2094 static void free_pg_vec(char **pg_vec, unsigned int order, unsigned int len)
2095 {
2096         int i;
2097
2098         for (i = 0; i < len; i++) {
2099                 if (likely(pg_vec[i]))
2100                         free_pages((unsigned long) pg_vec[i], order);
2101         }
2102         kfree(pg_vec);
2103 }
2104
2105 static inline char *alloc_one_pg_vec_page(unsigned long order)
2106 {
2107         gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | __GFP_ZERO | __GFP_NOWARN;
2108
2109         return (char *) __get_free_pages(gfp_flags, order);
2110 }
2111
2112 static char **alloc_pg_vec(struct tpacket_req *req, int order)
2113 {
2114         unsigned int block_nr = req->tp_block_nr;
2115         char **pg_vec;
2116         int i;
2117
2118         pg_vec = kzalloc(block_nr * sizeof(char *), GFP_KERNEL);
2119         if (unlikely(!pg_vec))
2120                 goto out;
2121
2122         for (i = 0; i < block_nr; i++) {
2123                 pg_vec[i] = alloc_one_pg_vec_page(order);
2124                 if (unlikely(!pg_vec[i]))
2125                         goto out_free_pgvec;
2126         }
2127
2128 out:
2129         return pg_vec;
2130
2131 out_free_pgvec:
2132         free_pg_vec(pg_vec, order, block_nr);
2133         pg_vec = NULL;
2134         goto out;
2135 }
2136
2137 static int packet_set_ring(struct sock *sk, struct tpacket_req *req,
2138                 int closing, int tx_ring)
2139 {
2140         char **pg_vec = NULL;
2141         struct packet_sock *po = pkt_sk(sk);
2142         int was_running, order = 0;
2143         struct packet_ring_buffer *rb;
2144         struct sk_buff_head *rb_queue;
2145         __be16 num;
2146         int err;
2147
2148         rb = tx_ring ? &po->tx_ring : &po->rx_ring;
2149         rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
2150
2151         err = -EBUSY;
2152         if (!closing) {
2153                 if (atomic_read(&po->mapped))
2154                         goto out;
2155                 if (atomic_read(&rb->pending))
2156                         goto out;
2157         }
2158
2159         if (req->tp_block_nr) {
2160                 /* Sanity tests and some calculations */
2161                 err = -EBUSY;
2162                 if (unlikely(rb->pg_vec))
2163                         goto out;
2164
2165                 switch (po->tp_version) {
2166                 case TPACKET_V1:
2167                         po->tp_hdrlen = TPACKET_HDRLEN;
2168                         break;
2169                 case TPACKET_V2:
2170                         po->tp_hdrlen = TPACKET2_HDRLEN;
2171                         break;
2172                 }
2173
2174                 err = -EINVAL;
2175                 if (unlikely((int)req->tp_block_size <= 0))
2176                         goto out;
2177                 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
2178                         goto out;
2179                 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
2180                                         po->tp_reserve))
2181                         goto out;
2182                 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
2183                         goto out;
2184
2185                 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
2186                 if (unlikely(rb->frames_per_block <= 0))
2187                         goto out;
2188                 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
2189                                         req->tp_frame_nr))
2190                         goto out;
2191
2192                 err = -ENOMEM;
2193                 order = get_order(req->tp_block_size);
2194                 pg_vec = alloc_pg_vec(req, order);
2195                 if (unlikely(!pg_vec))
2196                         goto out;
2197         }
2198         /* Done */
2199         else {
2200                 err = -EINVAL;
2201                 if (unlikely(req->tp_frame_nr))
2202                         goto out;
2203         }
2204
2205         lock_sock(sk);
2206
2207         /* Detach socket from network */
2208         spin_lock(&po->bind_lock);
2209         was_running = po->running;
2210         num = po->num;
2211         if (was_running) {
2212                 __dev_remove_pack(&po->prot_hook);
2213                 po->num = 0;
2214                 po->running = 0;
2215                 __sock_put(sk);
2216         }
2217         spin_unlock(&po->bind_lock);
2218
2219         synchronize_net();
2220
2221         err = -EBUSY;
2222         mutex_lock(&po->pg_vec_lock);
2223         if (closing || atomic_read(&po->mapped) == 0) {
2224                 err = 0;
2225 #define XC(a, b) ({ __typeof__ ((a)) __t; __t = (a); (a) = (b); __t; })
2226                 spin_lock_bh(&rb_queue->lock);
2227                 pg_vec = XC(rb->pg_vec, pg_vec);
2228                 rb->frame_max = (req->tp_frame_nr - 1);
2229                 rb->head = 0;
2230                 rb->frame_size = req->tp_frame_size;
2231                 spin_unlock_bh(&rb_queue->lock);
2232
2233                 order = XC(rb->pg_vec_order, order);
2234                 req->tp_block_nr = XC(rb->pg_vec_len, req->tp_block_nr);
2235
2236                 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
2237                 po->prot_hook.func = (po->rx_ring.pg_vec) ?
2238                                                 tpacket_rcv : packet_rcv;
2239                 skb_queue_purge(rb_queue);
2240 #undef XC
2241                 if (atomic_read(&po->mapped))
2242                         pr_err("packet_mmap: vma is busy: %d\n",
2243                                atomic_read(&po->mapped));
2244         }
2245         mutex_unlock(&po->pg_vec_lock);
2246
2247         spin_lock(&po->bind_lock);
2248         if (was_running && !po->running) {
2249                 sock_hold(sk);
2250                 po->running = 1;
2251                 po->num = num;
2252                 dev_add_pack(&po->prot_hook);
2253         }
2254         spin_unlock(&po->bind_lock);
2255
2256         release_sock(sk);
2257
2258         if (pg_vec)
2259                 free_pg_vec(pg_vec, order, req->tp_block_nr);
2260 out:
2261         return err;
2262 }
2263
2264 static int packet_mmap(struct file *file, struct socket *sock,
2265                 struct vm_area_struct *vma)
2266 {
2267         struct sock *sk = sock->sk;
2268         struct packet_sock *po = pkt_sk(sk);
2269         unsigned long size, expected_size;
2270         struct packet_ring_buffer *rb;
2271         unsigned long start;
2272         int err = -EINVAL;
2273         int i;
2274
2275         if (vma->vm_pgoff)
2276                 return -EINVAL;
2277
2278         mutex_lock(&po->pg_vec_lock);
2279
2280         expected_size = 0;
2281         for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
2282                 if (rb->pg_vec) {
2283                         expected_size += rb->pg_vec_len
2284                                                 * rb->pg_vec_pages
2285                                                 * PAGE_SIZE;
2286                 }
2287         }
2288
2289         if (expected_size == 0)
2290                 goto out;
2291
2292         size = vma->vm_end - vma->vm_start;
2293         if (size != expected_size)
2294                 goto out;
2295
2296         start = vma->vm_start;
2297         for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
2298                 if (rb->pg_vec == NULL)
2299                         continue;
2300
2301                 for (i = 0; i < rb->pg_vec_len; i++) {
2302                         struct page *page = virt_to_page(rb->pg_vec[i]);
2303                         int pg_num;
2304
2305                         for (pg_num = 0; pg_num < rb->pg_vec_pages;
2306                                         pg_num++, page++) {
2307                                 err = vm_insert_page(vma, start, page);
2308                                 if (unlikely(err))
2309                                         goto out;
2310                                 start += PAGE_SIZE;
2311                         }
2312                 }
2313         }
2314
2315         atomic_inc(&po->mapped);
2316         vma->vm_ops = &packet_mmap_ops;
2317         err = 0;
2318
2319 out:
2320         mutex_unlock(&po->pg_vec_lock);
2321         return err;
2322 }
2323 #endif
2324
2325
2326 static const struct proto_ops packet_ops_spkt = {
2327         .family =       PF_PACKET,
2328         .owner =        THIS_MODULE,
2329         .release =      packet_release,
2330         .bind =         packet_bind_spkt,
2331         .connect =      sock_no_connect,
2332         .socketpair =   sock_no_socketpair,
2333         .accept =       sock_no_accept,
2334         .getname =      packet_getname_spkt,
2335         .poll =         datagram_poll,
2336         .ioctl =        packet_ioctl,
2337         .listen =       sock_no_listen,
2338         .shutdown =     sock_no_shutdown,
2339         .setsockopt =   sock_no_setsockopt,
2340         .getsockopt =   sock_no_getsockopt,
2341         .sendmsg =      packet_sendmsg_spkt,
2342         .recvmsg =      packet_recvmsg,
2343         .mmap =         sock_no_mmap,
2344         .sendpage =     sock_no_sendpage,
2345 };
2346
2347 static const struct proto_ops packet_ops = {
2348         .family =       PF_PACKET,
2349         .owner =        THIS_MODULE,
2350         .release =      packet_release,
2351         .bind =         packet_bind,
2352         .connect =      sock_no_connect,
2353         .socketpair =   sock_no_socketpair,
2354         .accept =       sock_no_accept,
2355         .getname =      packet_getname,
2356         .poll =         packet_poll,
2357         .ioctl =        packet_ioctl,
2358         .listen =       sock_no_listen,
2359         .shutdown =     sock_no_shutdown,
2360         .setsockopt =   packet_setsockopt,
2361         .getsockopt =   packet_getsockopt,
2362         .sendmsg =      packet_sendmsg,
2363         .recvmsg =      packet_recvmsg,
2364         .mmap =         packet_mmap,
2365         .sendpage =     sock_no_sendpage,
2366 };
2367
2368 static const struct net_proto_family packet_family_ops = {
2369         .family =       PF_PACKET,
2370         .create =       packet_create,
2371         .owner  =       THIS_MODULE,
2372 };
2373
2374 static struct notifier_block packet_netdev_notifier = {
2375         .notifier_call =        packet_notifier,
2376 };
2377
2378 #ifdef CONFIG_PROC_FS
2379 static inline struct sock *packet_seq_idx(struct net *net, loff_t off)
2380 {
2381         struct sock *s;
2382         struct hlist_node *node;
2383
2384         sk_for_each(s, node, &net->packet.sklist) {
2385                 if (!off--)
2386                         return s;
2387         }
2388         return NULL;
2389 }
2390
2391 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
2392         __acquires(seq_file_net(seq)->packet.sklist_lock)
2393 {
2394         struct net *net = seq_file_net(seq);
2395         read_lock(&net->packet.sklist_lock);
2396         return *pos ? packet_seq_idx(net, *pos - 1) : SEQ_START_TOKEN;
2397 }
2398
2399 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2400 {
2401         struct net *net = seq_file_net(seq);
2402         ++*pos;
2403         return  (v == SEQ_START_TOKEN)
2404                 ? sk_head(&net->packet.sklist)
2405                 : sk_next((struct sock *)v) ;
2406 }
2407
2408 static void packet_seq_stop(struct seq_file *seq, void *v)
2409         __releases(seq_file_net(seq)->packet.sklist_lock)
2410 {
2411         struct net *net = seq_file_net(seq);
2412         read_unlock(&net->packet.sklist_lock);
2413 }
2414
2415 static int packet_seq_show(struct seq_file *seq, void *v)
2416 {
2417         if (v == SEQ_START_TOKEN)
2418                 seq_puts(seq, "sk       RefCnt Type Proto  Iface R Rmem   User   Inode\n");
2419         else {
2420                 struct sock *s = v;
2421                 const struct packet_sock *po = pkt_sk(s);
2422
2423                 seq_printf(seq,
2424                            "%p %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6lu\n",
2425                            s,
2426                            atomic_read(&s->sk_refcnt),
2427                            s->sk_type,
2428                            ntohs(po->num),
2429                            po->ifindex,
2430                            po->running,
2431                            atomic_read(&s->sk_rmem_alloc),
2432                            sock_i_uid(s),
2433                            sock_i_ino(s));
2434         }
2435
2436         return 0;
2437 }
2438
2439 static const struct seq_operations packet_seq_ops = {
2440         .start  = packet_seq_start,
2441         .next   = packet_seq_next,
2442         .stop   = packet_seq_stop,
2443         .show   = packet_seq_show,
2444 };
2445
2446 static int packet_seq_open(struct inode *inode, struct file *file)
2447 {
2448         return seq_open_net(inode, file, &packet_seq_ops,
2449                             sizeof(struct seq_net_private));
2450 }
2451
2452 static const struct file_operations packet_seq_fops = {
2453         .owner          = THIS_MODULE,
2454         .open           = packet_seq_open,
2455         .read           = seq_read,
2456         .llseek         = seq_lseek,
2457         .release        = seq_release_net,
2458 };
2459
2460 #endif
2461
2462 static int packet_net_init(struct net *net)
2463 {
2464         rwlock_init(&net->packet.sklist_lock);
2465         INIT_HLIST_HEAD(&net->packet.sklist);
2466
2467         if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops))
2468                 return -ENOMEM;
2469
2470         return 0;
2471 }
2472
2473 static void packet_net_exit(struct net *net)
2474 {
2475         proc_net_remove(net, "packet");
2476 }
2477
2478 static struct pernet_operations packet_net_ops = {
2479         .init = packet_net_init,
2480         .exit = packet_net_exit,
2481 };
2482
2483
2484 static void __exit packet_exit(void)
2485 {
2486         unregister_netdevice_notifier(&packet_netdev_notifier);
2487         unregister_pernet_subsys(&packet_net_ops);
2488         sock_unregister(PF_PACKET);
2489         proto_unregister(&packet_proto);
2490 }
2491
2492 static int __init packet_init(void)
2493 {
2494         int rc = proto_register(&packet_proto, 0);
2495
2496         if (rc != 0)
2497                 goto out;
2498
2499         sock_register(&packet_family_ops);
2500         register_pernet_subsys(&packet_net_ops);
2501         register_netdevice_notifier(&packet_netdev_notifier);
2502 out:
2503         return rc;
2504 }
2505
2506 module_init(packet_init);
2507 module_exit(packet_exit);
2508 MODULE_LICENSE("GPL");
2509 MODULE_ALIAS_NETPROTO(PF_PACKET);