tcp: Skip empty hash buckets faster in /proc/net/tcp

[safe/jmp/linux-2.6] / net / ipv4 / tcp_ipv4.c

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 5400d75..37ca384 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -418,7 +418,7 @@ void tcp_v4_err(struct sk_buff *skb, u32 info)
                /* ICMPs are not backlogged, hence we cannot get
                   an established socket here.
                 */
-               BUG_TRAP(!req->sk);
+               WARN_ON(req->sk);
 
                if (seq != tcp_rsk(req)->snt_isn) {
                        NET_INC_STATS_BH(net, LINUX_MIB_OUTOFWINDOWICMPS);
@@ -655,8 +655,8 @@ static void tcp_v4_send_ack(struct sk_buff *skb, u32 seq, u32 ack,
                rep.th.doff = arg.iov[0].iov_len/4;
 
                tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
-                                   key, ip_hdr(skb)->daddr,
-                                   ip_hdr(skb)->saddr, &rep.th);
+                                   key, ip_hdr(skb)->saddr,
+                                   ip_hdr(skb)->daddr, &rep.th);
        }
 #endif
        arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
@@ -687,14 +687,14 @@ static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
        inet_twsk_put(tw);
 }
 
-static void tcp_v4_reqsk_send_ack(struct sk_buff *skb,
+static void tcp_v4_reqsk_send_ack(struct sock *sk, struct sk_buff *skb,
                                  struct request_sock *req)
 {
        tcp_v4_send_ack(skb, tcp_rsk(req)->snt_isn + 1,
                        tcp_rsk(req)->rcv_isn + 1, req->rcv_wnd,
                        req->ts_recent,
                        0,
-                       tcp_v4_md5_do_lookup(skb->sk, ip_hdr(skb)->daddr));
+                       tcp_v4_md5_do_lookup(sk, ip_hdr(skb)->daddr));
 }
 
 /*
@@ -1116,18 +1116,12 @@ static int tcp_v4_inbound_md5_hash(struct sock *sk, struct sk_buff *skb)
                return 0;
 
        if (hash_expected && !hash_location) {
-               LIMIT_NETDEBUG(KERN_INFO "MD5 Hash expected but NOT found "
-                              "(" NIPQUAD_FMT ", %d)->(" NIPQUAD_FMT ", %d)\n",
-                              NIPQUAD(iph->saddr), ntohs(th->source),
-                              NIPQUAD(iph->daddr), ntohs(th->dest));
+               NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
                return 1;
        }
 
        if (!hash_expected && hash_location) {
-               LIMIT_NETDEBUG(KERN_INFO "MD5 Hash NOT expected but found "
-                              "(" NIPQUAD_FMT ", %d)->(" NIPQUAD_FMT ", %d)\n",
-                              NIPQUAD(iph->saddr), ntohs(th->source),
-                              NIPQUAD(iph->daddr), ntohs(th->dest));
+               NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
                return 1;
        }
 
@@ -1892,8 +1886,7 @@ static void *listening_get_next(struct seq_file *seq, void *cur)
                req = req->dl_next;
                while (1) {
                        while (req) {
-                               if (req->rsk_ops->family == st->family &&
-                                   net_eq(sock_net(req->sk), net)) {
+                               if (req->rsk_ops->family == st->family) {
                                        cur = req;
                                        goto out;
                                }
@@ -1953,6 +1946,12 @@ static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
        return rc;
 }
 
+static inline int empty_bucket(struct tcp_iter_state *st)
+{
+       return hlist_empty(&tcp_hashinfo.ehash[st->bucket].chain) &&
+               hlist_empty(&tcp_hashinfo.ehash[st->bucket].twchain);
+}
+
 static void *established_get_first(struct seq_file *seq)
 {
        struct tcp_iter_state* st = seq->private;
@@ -1965,6 +1964,10 @@ static void *established_get_first(struct seq_file *seq)
                struct inet_timewait_sock *tw;
                rwlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
 
+               /* Lockless fast path for the common case of empty buckets */
+               if (empty_bucket(st))
+                       continue;
+
                read_lock_bh(lock);
                sk_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
                        if (sk->sk_family != st->family ||
@@ -2015,13 +2018,15 @@ get_tw:
                read_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
                st->state = TCP_SEQ_STATE_ESTABLISHED;
 
-               if (++st->bucket < tcp_hashinfo.ehash_size) {
-                       read_lock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
-                       sk = sk_head(&tcp_hashinfo.ehash[st->bucket].chain);
-               } else {
-                       cur = NULL;
-                       goto out;
-               }
+               /* Look for next non empty bucket */
+               while (++st->bucket < tcp_hashinfo.ehash_size &&
+                               empty_bucket(st))
+                       ;
+               if (st->bucket >= tcp_hashinfo.ehash_size)
+                       return NULL;
+
+               read_lock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
+               sk = sk_head(&tcp_hashinfo.ehash[st->bucket].chain);
        } else
                sk = sk_next(sk);