net: speedup udp receive path

Since commit 95766fff ([UDP]: Add memory accounting.),
each received packet needs one extra sock_lock()/sock_release() pair.

This added latency because of possible backlog handling. Then later,
ticket spinlocks added yet another latency source in case of DDOS.

This patch introduces lock_sock_bh() and unlock_sock_bh()
synchronization primitives, avoiding one atomic operation and backlog
processing.

skb_free_datagram_locked() uses them instead of full blown
lock_sock()/release_sock(). skb is orphaned inside locked section for
proper socket memory reclaim, and finally freed outside of it.

UDP receive path now take the socket spinlock only once.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/sock.h b/include/net/sock.h
index cf12b1e..d361c77 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1021,6 +1021,16 @@ extern void release_sock(struct sock *sk);
                                SINGLE_DEPTH_NESTING)
 #define bh_unlock_sock(__sk)   spin_unlock(&((__sk)->sk_lock.slock))
 
+static inline void lock_sock_bh(struct sock *sk)
+{
+       spin_lock_bh(&sk->sk_lock.slock);
+}
+
+static inline void unlock_sock_bh(struct sock *sk)
+{
+       spin_unlock_bh(&sk->sk_lock.slock);
+}
+
 extern struct sock             *sk_alloc(struct net *net, int family,
                                          gfp_t priority,
                                          struct proto *prot);

diff --git a/net/core/datagram.c b/net/core/datagram.c
index 5574a5d..95b851f 100644 (file)
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -229,9 +229,13 @@ EXPORT_SYMBOL(skb_free_datagram);
 
 void skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb)
 {
-       lock_sock(sk);
-       skb_free_datagram(sk, skb);
-       release_sock(sk);
+       lock_sock_bh(sk);
+       skb_orphan(skb);
+       sk_mem_reclaim_partial(sk);
+       unlock_sock_bh(sk);
+
+       /* skb is now orphaned, might be freed outside of locked section */
+       consume_skb(skb);
 }
 EXPORT_SYMBOL(skb_free_datagram_locked);
 

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 63eb56b..1f86965 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1062,10 +1062,10 @@ static unsigned int first_packet_length(struct sock *sk)
        spin_unlock_bh(&rcvq->lock);
 
        if (!skb_queue_empty(&list_kill)) {
-               lock_sock(sk);
+               lock_sock_bh(sk);
                __skb_queue_purge(&list_kill);
                sk_mem_reclaim_partial(sk);
-               release_sock(sk);
+               unlock_sock_bh(sk);
        }
        return res;
 }
@@ -1196,10 +1196,10 @@ out:
        return err;
 
 csum_copy_err:
-       lock_sock(sk);
+       lock_sock_bh(sk);
        if (!skb_kill_datagram(sk, skb, flags))
                UDP_INC_STATS_USER(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
-       release_sock(sk);
+       unlock_sock_bh(sk);
 
        if (noblock)
                return -EAGAIN;
@@ -1624,9 +1624,9 @@ int udp_rcv(struct sk_buff *skb)
 
 void udp_destroy_sock(struct sock *sk)
 {
-       lock_sock(sk);
+       lock_sock_bh(sk);
        udp_flush_pending_frames(sk);
-       release_sock(sk);
+       unlock_sock_bh(sk);
 }
 
 /*

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 3ead20a..91c60f0 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -424,7 +424,7 @@ out:
        return err;
 
 csum_copy_err:
-       lock_sock(sk);
+       lock_sock_bh(sk);
        if (!skb_kill_datagram(sk, skb, flags)) {
                if (is_udp4)
                        UDP_INC_STATS_USER(sock_net(sk),
@@ -433,7 +433,7 @@ csum_copy_err:
                        UDP6_INC_STATS_USER(sock_net(sk),
                                        UDP_MIB_INERRORS, is_udplite);
        }
-       release_sock(sk);
+       unlock_sock_bh(sk);
 
        if (flags & MSG_DONTWAIT)
                return -EAGAIN;