net: spread __net_init, __net_exit

[safe/jmp/linux-2.6] / net / packet / af_packet.c

diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index d398a9b..a97acfe 100644 (file)
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -79,6 +79,7 @@
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/mutex.h>

+#include <linux/if_vlan.h>

 
 #ifdef CONFIG_INET
 #include <net/inet_common.h>
 
 #ifdef CONFIG_INET
 #include <net/inet_common.h>


@@ -188,7 +189,6 @@ struct packet_sock {
        struct packet_ring_buffer       tx_ring;
        int                     copy_thresh;
 #endif
        struct packet_ring_buffer       tx_ring;
        int                     copy_thresh;
 #endif

-       struct packet_type      prot_hook;

        spinlock_t              bind_lock;
        struct mutex            pg_vec_lock;
        unsigned int            running:1,      /* prot_hook is attached*/
        spinlock_t              bind_lock;
        struct mutex            pg_vec_lock;
        unsigned int            running:1,      /* prot_hook is attached*/


@@ -204,6 +204,7 @@ struct packet_sock {
        unsigned int            tp_reserve;
        unsigned int            tp_loss:1;
 #endif
        unsigned int            tp_reserve;
        unsigned int            tp_loss:1;
 #endif

+       struct packet_type      prot_hook ____cacheline_aligned_in_smp;

 };
 
 struct packet_skb_cb {
 };
 
 struct packet_skb_cb {


@@ -364,7 +365,7 @@ static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
        if (skb->pkt_type == PACKET_LOOPBACK)
                goto out;
 
        if (skb->pkt_type == PACKET_LOOPBACK)
                goto out;
 

-       if (dev_net(dev) != sock_net(sk))
+       if (!net_eq(dev_net(dev), sock_net(sk)))

                goto out;
 
        skb = skb_share_check(skb, GFP_ATOMIC);
                goto out;
 
        skb = skb_share_check(skb, GFP_ATOMIC);


@@ -414,7 +415,7 @@ static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
 {
        struct sock *sk = sock->sk;
        struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
 {
        struct sock *sk = sock->sk;
        struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;

-       struct sk_buff *skb;
+       struct sk_buff *skb = NULL;

        struct net_device *dev;
        __be16 proto = 0;
        int err;
        struct net_device *dev;
        __be16 proto = 0;
        int err;


@@ -436,7 +437,9 @@ static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
         */
 
        saddr->spkt_device[13] = 0;
         */
 
        saddr->spkt_device[13] = 0;

-       dev = dev_get_by_name(sock_net(sk), saddr->spkt_device);
+retry:
+       rcu_read_lock();
+       dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);

        err = -ENODEV;
        if (dev == NULL)
                goto out_unlock;
        err = -ENODEV;
        if (dev == NULL)
                goto out_unlock;


@@ -454,58 +457,48 @@ static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
        if (len > dev->mtu + dev->hard_header_len)
                goto out_unlock;
 
        if (len > dev->mtu + dev->hard_header_len)
                goto out_unlock;
 

-       err = -ENOBUFS;
-       skb = sock_wmalloc(sk, len + LL_RESERVED_SPACE(dev), 0, GFP_KERNEL);
-
-       /*
-        * If the write buffer is full, then tough. At this level the user
-        * gets to deal with the problem - do your own algorithmic backoffs.
-        * That's far more flexible.
-        */
-
-       if (skb == NULL)
-               goto out_unlock;
-
-       /*
-        *      Fill it in
-        */
-
-       /* FIXME: Save some space for broken drivers that write a
-        * hard header at transmission time by themselves. PPP is the
-        * notable one here. This should really be fixed at the driver level.
-        */
-       skb_reserve(skb, LL_RESERVED_SPACE(dev));
-       skb_reset_network_header(skb);
-
-       /* Try to align data part correctly */
-       if (dev->header_ops) {
-               skb->data -= dev->hard_header_len;
-               skb->tail -= dev->hard_header_len;
-               if (len < dev->hard_header_len)
-                       skb_reset_network_header(skb);
+       if (!skb) {
+               size_t reserved = LL_RESERVED_SPACE(dev);
+               unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
+
+               rcu_read_unlock();
+               skb = sock_wmalloc(sk, len + reserved, 0, GFP_KERNEL);
+               if (skb == NULL)
+                       return -ENOBUFS;
+               /* FIXME: Save some space for broken drivers that write a hard
+                * header at transmission time by themselves. PPP is the notable
+                * one here. This should really be fixed at the driver level.
+                */
+               skb_reserve(skb, reserved);
+               skb_reset_network_header(skb);
+
+               /* Try to align data part correctly */
+               if (hhlen) {
+                       skb->data -= hhlen;
+                       skb->tail -= hhlen;
+                       if (len < hhlen)
+                               skb_reset_network_header(skb);
+               }
+               err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
+               if (err)
+                       goto out_free;
+               goto retry;

        }
 
        }
 

-       /* Returns -EFAULT on error */
-       err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
+

        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = sk->sk_priority;
        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = sk->sk_priority;

-       if (err)
-               goto out_free;
-
-       /*
-        *      Now send it
-        */
+       skb->mark = sk->sk_mark;

 
        dev_queue_xmit(skb);
 
        dev_queue_xmit(skb);

-       dev_put(dev);
+       rcu_read_unlock();

        return len;
 
        return len;
 

+out_unlock:
+       rcu_read_unlock();

 out_free:
        kfree_skb(skb);
 out_free:
        kfree_skb(skb);

-out_unlock:
-       if (dev)
-               dev_put(dev);

        return err;
 }
 
        return err;
 }
 


@@ -524,31 +517,6 @@ static inline unsigned int run_filter(struct sk_buff *skb, struct sock *sk,
 }
 
 /*
 }
 
 /*

- * If we've lost frames since the last time we queued one to the
- * sk_receive_queue, we need to record it here.
- * This must be called under the protection of the socket lock
- * to prevent racing with other softirqs and user space
- */
-static inline void record_packet_gap(struct sk_buff *skb,
-                                       struct packet_sock *po)
-{
-       /*
-        * We overload the mark field here, since we're about
-        * to enqueue to a receive queue and no body else will
-        * use this field at this point
-        */
-       skb->mark = po->stats.tp_gap;
-       po->stats.tp_gap = 0;
-       return;
-
-}
-
-static inline __u32 check_packet_gap(struct sk_buff *skb)
-{
-       return skb->mark;
-}
-
-/*

    This function makes lazy skb cloning in hope that most of packets
    are discarded by BPF.
 
    This function makes lazy skb cloning in hope that most of packets
    are discarded by BPF.
 


@@ -576,7 +544,7 @@ static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
        sk = pt->af_packet_priv;
        po = pkt_sk(sk);
 
        sk = pt->af_packet_priv;
        po = pkt_sk(sk);
 

-       if (dev_net(dev) != sock_net(sk))
+       if (!net_eq(dev_net(dev), sock_net(sk)))

                goto drop;
 
        skb->dev = dev;
                goto drop;
 
        skb->dev = dev;


@@ -651,17 +619,14 @@ static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
 
        spin_lock(&sk->sk_receive_queue.lock);
        po->stats.tp_packets++;
 
        spin_lock(&sk->sk_receive_queue.lock);
        po->stats.tp_packets++;

-       record_packet_gap(skb, po);
+       skb->dropcount = atomic_read(&sk->sk_drops);

        __skb_queue_tail(&sk->sk_receive_queue, skb);
        spin_unlock(&sk->sk_receive_queue.lock);
        sk->sk_data_ready(sk, skb->len);
        return 0;
 
 drop_n_acct:
        __skb_queue_tail(&sk->sk_receive_queue, skb);
        spin_unlock(&sk->sk_receive_queue.lock);
        sk->sk_data_ready(sk, skb->len);
        return 0;
 
 drop_n_acct:

-       spin_lock(&sk->sk_receive_queue.lock);
-       po->stats.tp_drops++;
-       po->stats.tp_gap++;
-       spin_unlock(&sk->sk_receive_queue.lock);
+       po->stats.tp_drops = atomic_inc_return(&sk->sk_drops);

 
 drop_n_restore:
        if (skb_head != skb->data && skb_shared(skb)) {
 
 drop_n_restore:
        if (skb_head != skb->data && skb_shared(skb)) {


@@ -700,7 +665,7 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
        sk = pt->af_packet_priv;
        po = pkt_sk(sk);
 
        sk = pt->af_packet_priv;
        po = pkt_sk(sk);
 

-       if (dev_net(dev) != sock_net(sk))
+       if (!net_eq(dev_net(dev), sock_net(sk)))

                goto drop;
 
        if (dev->header_ops) {
                goto drop;
 
        if (dev->header_ops) {


@@ -793,7 +758,7 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
                        getnstimeofday(&ts);
                h.h2->tp_sec = ts.tv_sec;
                h.h2->tp_nsec = ts.tv_nsec;
                        getnstimeofday(&ts);
                h.h2->tp_sec = ts.tv_sec;
                h.h2->tp_nsec = ts.tv_nsec;

-               h.h2->tp_vlan_tci = skb->vlan_tci;
+               h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);

                hdrlen = sizeof(*h.h2);
                break;
        default:
                hdrlen = sizeof(*h.h2);
                break;
        default:


@@ -838,7 +803,6 @@ drop:
 
 ring_is_full:
        po->stats.tp_drops++;
 
 ring_is_full:
        po->stats.tp_drops++;

-       po->stats.tp_gap++;

        spin_unlock(&sk->sk_receive_queue.lock);
 
        sk->sk_data_ready(sk, 0);
        spin_unlock(&sk->sk_receive_queue.lock);
 
        sk->sk_data_ready(sk, 0);


@@ -884,6 +848,7 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = po->sk.sk_priority;
        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = po->sk.sk_priority;

+       skb->mark = po->sk.sk_mark;

        skb_shinfo(skb)->destructor_arg = ph.raw;
 
        switch (po->tp_version) {
        skb_shinfo(skb)->destructor_arg = ph.raw;
 
        switch (po->tp_version) {


@@ -1010,10 +975,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
                goto out_put;
 
        size_max = po->tx_ring.frame_size
                goto out_put;
 
        size_max = po->tx_ring.frame_size

-               - sizeof(struct skb_shared_info)
-               - po->tp_hdrlen
-               - LL_ALLOCATED_SPACE(dev)
-               - sizeof(struct sockaddr_ll);
+               - (po->tp_hdrlen - sizeof(struct sockaddr_ll));

 
        if (size_max > dev->mtu + reserve)
                size_max = dev->mtu + reserve;
 
        if (size_max > dev->mtu + reserve)
                size_max = dev->mtu + reserve;


@@ -1063,9 +1025,10 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
                        goto out_xmit;
                packet_increment_head(&po->tx_ring);
                len_sum += tp_len;
                        goto out_xmit;
                packet_increment_head(&po->tx_ring);
                len_sum += tp_len;

-       } while (likely((ph != NULL) || ((!(msg->msg_flags & MSG_DONTWAIT))
-                                       && (atomic_read(&po->tx_ring.pending))))
-             );
+       } while (likely((ph != NULL) ||
+                       ((!(msg->msg_flags & MSG_DONTWAIT)) &&
+                        (atomic_read(&po->tx_ring.pending))))
+               );

 
        err = len_sum;
        goto out_put;
 
        err = len_sum;
        goto out_put;


@@ -1153,6 +1116,7 @@ static int packet_snd(struct socket *sock,
        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = sk->sk_priority;
        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = sk->sk_priority;

+       skb->mark = sk->sk_mark;

 
        /*
         *      Now send it
 
        /*
         *      Now send it


@@ -1372,7 +1336,8 @@ static struct proto packet_proto = {
  *     Create a packet of type SOCK_PACKET.
  */
 
  *     Create a packet of type SOCK_PACKET.
  */
 

-static int packet_create(struct net *net, struct socket *sock, int protocol)
+static int packet_create(struct net *net, struct socket *sock, int protocol,
+                        int kern)

 {
        struct sock *sk;
        struct packet_sock *po;
 {
        struct sock *sk;
        struct packet_sock *po;


@@ -1446,7 +1411,6 @@ static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
        struct sk_buff *skb;
        int copied, err;
        struct sockaddr_ll *sll;
        struct sk_buff *skb;
        int copied, err;
        struct sockaddr_ll *sll;

-       __u32 gap;

 
        err = -EINVAL;
        if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT))
 
        err = -EINVAL;
        if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT))


@@ -1504,7 +1468,7 @@ static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
        if (err)
                goto out_free;
 
        if (err)
                goto out_free;
 

-       sock_recv_timestamp(msg, sk, skb);
+       sock_recv_ts_and_drops(msg, sk, skb);

 
        if (msg->msg_name)
                memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
 
        if (msg->msg_name)
                memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,


@@ -1520,15 +1484,11 @@ static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
                aux.tp_snaplen = skb->len;
                aux.tp_mac = 0;
                aux.tp_net = skb_network_offset(skb);
                aux.tp_snaplen = skb->len;
                aux.tp_mac = 0;
                aux.tp_net = skb_network_offset(skb);

-               aux.tp_vlan_tci = skb->vlan_tci;
+               aux.tp_vlan_tci = vlan_tx_tag_get(skb);

 
                put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
        }
 
 
                put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
        }
 

-       gap = check_packet_gap(skb);
-       if (gap)
-               put_cmsg(msg, SOL_PACKET, PACKET_GAPDATA, sizeof(__u32), &gap);
-

        /*
         *      Free or return the buffer as appropriate. Again this
         *      hides all the races and re-entrancy issues from us.
        /*
         *      Free or return the buffer as appropriate. Again this
         *      hides all the races and re-entrancy issues from us.


@@ -1551,12 +1511,13 @@ static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
                return -EOPNOTSUPP;
 
        uaddr->sa_family = AF_PACKET;
                return -EOPNOTSUPP;
 
        uaddr->sa_family = AF_PACKET;

-       dev = dev_get_by_index(sock_net(sk), pkt_sk(sk)->ifindex);
-       if (dev) {
+       rcu_read_lock();
+       dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
+       if (dev)

                strlcpy(uaddr->sa_data, dev->name, 15);
                strlcpy(uaddr->sa_data, dev->name, 15);

-               dev_put(dev);
-       } else
+       else

                memset(uaddr->sa_data, 0, 14);
                memset(uaddr->sa_data, 0, 14);

+       rcu_read_unlock();

        *uaddr_len = sizeof(*uaddr);
 
        return 0;
        *uaddr_len = sizeof(*uaddr);
 
        return 0;


@@ -1568,7 +1529,7 @@ static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
        struct net_device *dev;
        struct sock *sk = sock->sk;
        struct packet_sock *po = pkt_sk(sk);
        struct net_device *dev;
        struct sock *sk = sock->sk;
        struct packet_sock *po = pkt_sk(sk);

-       struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
+       DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);

 
        if (peer)
                return -EOPNOTSUPP;
 
        if (peer)
                return -EOPNOTSUPP;


@@ -1576,16 +1537,17 @@ static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
        sll->sll_family = AF_PACKET;
        sll->sll_ifindex = po->ifindex;
        sll->sll_protocol = po->num;
        sll->sll_family = AF_PACKET;
        sll->sll_ifindex = po->ifindex;
        sll->sll_protocol = po->num;

-       dev = dev_get_by_index(sock_net(sk), po->ifindex);
+       rcu_read_lock();
+       dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);

        if (dev) {
                sll->sll_hatype = dev->type;
                sll->sll_halen = dev->addr_len;
                memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
        if (dev) {
                sll->sll_hatype = dev->type;
                sll->sll_halen = dev->addr_len;
                memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);

-               dev_put(dev);

        } else {
                sll->sll_hatype = 0;    /* Bad: we have no ARPHRD_UNSPEC */
                sll->sll_halen = 0;
        }
        } else {
                sll->sll_hatype = 0;    /* Bad: we have no ARPHRD_UNSPEC */
                sll->sll_halen = 0;
        }

+       rcu_read_unlock();

        *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
 
        return 0;
        *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
 
        return 0;


@@ -1695,11 +1657,9 @@ static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
                        if (--ml->count == 0) {
                                struct net_device *dev;
                                *mlp = ml->next;
                        if (--ml->count == 0) {
                                struct net_device *dev;
                                *mlp = ml->next;

-                               dev = dev_get_by_index(sock_net(sk), ml->ifindex);
-                               if (dev) {
+                               dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
+                               if (dev)

                                        packet_dev_mc(dev, ml, -1);
                                        packet_dev_mc(dev, ml, -1);

-                                       dev_put(dev);
-                               }

                                kfree(ml);
                        }
                        rtnl_unlock();
                                kfree(ml);
                        }
                        rtnl_unlock();


@@ -1723,11 +1683,9 @@ static void packet_flush_mclist(struct sock *sk)
                struct net_device *dev;
 
                po->mclist = ml->next;
                struct net_device *dev;
 
                po->mclist = ml->next;

-               dev = dev_get_by_index(sock_net(sk), ml->ifindex);
-               if (dev != NULL) {
+               dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
+               if (dev != NULL)

                        packet_dev_mc(dev, ml, -1);
                        packet_dev_mc(dev, ml, -1);

-                       dev_put(dev);
-               }

                kfree(ml);
        }
        rtnl_unlock();
                kfree(ml);
        }
        rtnl_unlock();


@@ -2396,7 +2354,7 @@ static const struct proto_ops packet_ops = {
        .sendpage =     sock_no_sendpage,
 };
 
        .sendpage =     sock_no_sendpage,
 };
 

-static struct net_proto_family packet_family_ops = {
+static const struct net_proto_family packet_family_ops = {

        .family =       PF_PACKET,
        .create =       packet_create,
        .owner  =       THIS_MODULE,
        .family =       PF_PACKET,
        .create =       packet_create,
        .owner  =       THIS_MODULE,


@@ -2490,7 +2448,7 @@ static const struct file_operations packet_seq_fops = {
 
 #endif
 
 
 #endif
 

-static int packet_net_init(struct net *net)
+static int __net_init packet_net_init(struct net *net)

 {
        rwlock_init(&net->packet.sklist_lock);
        INIT_HLIST_HEAD(&net->packet.sklist);
 {
        rwlock_init(&net->packet.sklist_lock);
        INIT_HLIST_HEAD(&net->packet.sklist);


@@ -2501,7 +2459,7 @@ static int packet_net_init(struct net *net)
        return 0;
 }
 
        return 0;
 }
 

-static void packet_net_exit(struct net *net)
+static void __net_exit packet_net_exit(struct net *net)

 {
        proc_net_remove(net, "packet");
 }
 {
        proc_net_remove(net, "packet");
 }