2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * The Internet Protocol (IP) output module.
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Donald Becker, <becker@super.org>
11 * Alan Cox, <Alan.Cox@linux.org>
13 * Stefan Becker, <stefanb@yello.ping.de>
14 * Jorge Cwik, <jorge@laser.satlink.net>
15 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
16 * Hirokazu Takahashi, <taka@valinux.co.jp>
18 * See ip_input.c for original log
21 * Alan Cox : Missing nonblock feature in ip_build_xmit.
22 * Mike Kilburn : htons() missing in ip_build_xmit.
23 * Bradford Johnson: Fix faulty handling of some frames when
25 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit
26 * (in case if packet not accepted by
27 * output firewall rules)
28 * Mike McLagan : Routing by source
29 * Alexey Kuznetsov: use new route cache
30 * Andi Kleen: Fix broken PMTU recovery and remove
31 * some redundant tests.
32 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
33 * Andi Kleen : Replace ip_reply with ip_send_reply.
34 * Andi Kleen : Split fast and slow ip_build_xmit path
35 * for decreased register pressure on x86
36 * and more readibility.
37 * Marc Boucher : When call_out_firewall returns FW_QUEUE,
38 * silently drop skb instead of failing with -EPERM.
39 * Detlev Wengorz : Copy protocol for fragments.
40 * Hirokazu Takahashi: HW checksumming for outgoing UDP
42 * Hirokazu Takahashi: sendfile() on UDP works now.
45 #include <asm/uaccess.h>
46 #include <asm/system.h>
47 #include <linux/module.h>
48 #include <linux/types.h>
49 #include <linux/kernel.h>
51 #include <linux/string.h>
52 #include <linux/errno.h>
53 #include <linux/highmem.h>
55 #include <linux/socket.h>
56 #include <linux/sockios.h>
58 #include <linux/inet.h>
59 #include <linux/netdevice.h>
60 #include <linux/etherdevice.h>
61 #include <linux/proc_fs.h>
62 #include <linux/stat.h>
63 #include <linux/init.h>
67 #include <net/protocol.h>
68 #include <net/route.h>
70 #include <linux/skbuff.h>
74 #include <net/checksum.h>
75 #include <net/inetpeer.h>
76 #include <linux/igmp.h>
77 #include <linux/netfilter_ipv4.h>
78 #include <linux/netfilter_bridge.h>
79 #include <linux/mroute.h>
80 #include <linux/netlink.h>
81 #include <linux/tcp.h>
83 int sysctl_ip_default_ttl __read_mostly = IPDEFTTL;
85 /* Generate a checksum for an outgoing IP datagram. */
86 __inline__ void ip_send_check(struct iphdr *iph)
89 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
92 int __ip_local_out(struct sk_buff *skb)
94 struct iphdr *iph = ip_hdr(skb);
96 iph->tot_len = htons(skb->len);
98 return nf_hook(PF_INET, NF_INET_LOCAL_OUT, skb, NULL, skb_dst(skb)->dev,
102 int ip_local_out(struct sk_buff *skb)
106 err = __ip_local_out(skb);
107 if (likely(err == 1))
108 err = dst_output(skb);
112 EXPORT_SYMBOL_GPL(ip_local_out);
114 /* dev_loopback_xmit for use with netfilter. */
115 static int ip_dev_loopback_xmit(struct sk_buff *newskb)
117 skb_reset_mac_header(newskb);
118 __skb_pull(newskb, skb_network_offset(newskb));
119 newskb->pkt_type = PACKET_LOOPBACK;
120 newskb->ip_summed = CHECKSUM_UNNECESSARY;
121 WARN_ON(!skb_dst(newskb));
126 static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
128 int ttl = inet->uc_ttl;
131 ttl = dst_metric(dst, RTAX_HOPLIMIT);
136 * Add an ip header to a skbuff and send it out.
139 int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk,
140 __be32 saddr, __be32 daddr, struct ip_options *opt)
142 struct inet_sock *inet = inet_sk(sk);
143 struct rtable *rt = skb_rtable(skb);
146 /* Build the IP header. */
147 skb_push(skb, sizeof(struct iphdr) + (opt ? opt->optlen : 0));
148 skb_reset_network_header(skb);
152 iph->tos = inet->tos;
153 if (ip_dont_fragment(sk, &rt->u.dst))
154 iph->frag_off = htons(IP_DF);
157 iph->ttl = ip_select_ttl(inet, &rt->u.dst);
158 iph->daddr = rt->rt_dst;
159 iph->saddr = rt->rt_src;
160 iph->protocol = sk->sk_protocol;
161 ip_select_ident(iph, &rt->u.dst, sk);
163 if (opt && opt->optlen) {
164 iph->ihl += opt->optlen>>2;
165 ip_options_build(skb, opt, daddr, rt, 0);
168 skb->priority = sk->sk_priority;
169 skb->mark = sk->sk_mark;
172 return ip_local_out(skb);
175 EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);
177 static inline int ip_finish_output2(struct sk_buff *skb)
179 struct dst_entry *dst = skb_dst(skb);
180 struct rtable *rt = (struct rtable *)dst;
181 struct net_device *dev = dst->dev;
182 unsigned int hh_len = LL_RESERVED_SPACE(dev);
184 if (rt->rt_type == RTN_MULTICAST) {
185 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len);
186 } else if (rt->rt_type == RTN_BROADCAST)
187 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTBCAST, skb->len);
189 /* Be paranoid, rather than too clever. */
190 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
191 struct sk_buff *skb2;
193 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
199 skb_set_owner_w(skb2, skb->sk);
205 return neigh_hh_output(dst->hh, skb);
206 else if (dst->neighbour)
207 return dst->neighbour->output(skb);
210 printk(KERN_DEBUG "ip_finish_output2: No header cache and no neighbour!\n");
215 static inline int ip_skb_dst_mtu(struct sk_buff *skb)
217 struct inet_sock *inet = skb->sk ? inet_sk(skb->sk) : NULL;
219 return (inet && inet->pmtudisc == IP_PMTUDISC_PROBE) ?
220 skb_dst(skb)->dev->mtu : dst_mtu(skb_dst(skb));
223 static int ip_finish_output(struct sk_buff *skb)
225 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
226 /* Policy lookup after SNAT yielded a new policy */
227 if (skb_dst(skb)->xfrm != NULL) {
228 IPCB(skb)->flags |= IPSKB_REROUTED;
229 return dst_output(skb);
232 if (skb->len > ip_skb_dst_mtu(skb) && !skb_is_gso(skb))
233 return ip_fragment(skb, ip_finish_output2);
235 return ip_finish_output2(skb);
238 int ip_mc_output(struct sk_buff *skb)
240 struct sock *sk = skb->sk;
241 struct rtable *rt = skb_rtable(skb);
242 struct net_device *dev = rt->u.dst.dev;
245 * If the indicated interface is up and running, send the packet.
247 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
250 skb->protocol = htons(ETH_P_IP);
253 * Multicasts are looped back for other local users
256 if (rt->rt_flags&RTCF_MULTICAST) {
257 if ((!sk || inet_sk(sk)->mc_loop)
258 #ifdef CONFIG_IP_MROUTE
259 /* Small optimization: do not loopback not local frames,
260 which returned after forwarding; they will be dropped
261 by ip_mr_input in any case.
262 Note, that local frames are looped back to be delivered
265 This check is duplicated in ip_mr_input at the moment.
267 && ((rt->rt_flags&RTCF_LOCAL) || !(IPCB(skb)->flags&IPSKB_FORWARDED))
270 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
272 NF_HOOK(PF_INET, NF_INET_POST_ROUTING, newskb,
274 ip_dev_loopback_xmit);
277 /* Multicasts with ttl 0 must not go beyond the host */
279 if (ip_hdr(skb)->ttl == 0) {
285 if (rt->rt_flags&RTCF_BROADCAST) {
286 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
288 NF_HOOK(PF_INET, NF_INET_POST_ROUTING, newskb, NULL,
289 newskb->dev, ip_dev_loopback_xmit);
292 return NF_HOOK_COND(PF_INET, NF_INET_POST_ROUTING, skb, NULL, skb->dev,
294 !(IPCB(skb)->flags & IPSKB_REROUTED));
297 int ip_output(struct sk_buff *skb)
299 struct net_device *dev = skb_dst(skb)->dev;
301 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
304 skb->protocol = htons(ETH_P_IP);
306 return NF_HOOK_COND(PF_INET, NF_INET_POST_ROUTING, skb, NULL, dev,
308 !(IPCB(skb)->flags & IPSKB_REROUTED));
311 int ip_queue_xmit(struct sk_buff *skb, int ipfragok)
313 struct sock *sk = skb->sk;
314 struct inet_sock *inet = inet_sk(sk);
315 struct ip_options *opt = inet->opt;
319 /* Skip all of this if the packet is already routed,
320 * f.e. by something like SCTP.
322 rt = skb_rtable(skb);
326 /* Make sure we can route this packet. */
327 rt = (struct rtable *)__sk_dst_check(sk, 0);
331 /* Use correct destination address if we have options. */
332 daddr = inet->inet_daddr;
337 struct flowi fl = { .oif = sk->sk_bound_dev_if,
341 .saddr = inet->inet_saddr,
342 .tos = RT_CONN_FLAGS(sk) } },
343 .proto = sk->sk_protocol,
344 .flags = inet_sk_flowi_flags(sk),
346 { .sport = inet->inet_sport,
347 .dport = inet->inet_dport } } };
349 /* If this fails, retransmit mechanism of transport layer will
350 * keep trying until route appears or the connection times
353 security_sk_classify_flow(sk, &fl);
354 if (ip_route_output_flow(sock_net(sk), &rt, &fl, sk, 0))
357 sk_setup_caps(sk, &rt->u.dst);
359 skb_dst_set(skb, dst_clone(&rt->u.dst));
362 if (opt && opt->is_strictroute && rt->rt_dst != rt->rt_gateway)
365 /* OK, we know where to send it, allocate and build IP header. */
366 skb_push(skb, sizeof(struct iphdr) + (opt ? opt->optlen : 0));
367 skb_reset_network_header(skb);
369 *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
370 if (ip_dont_fragment(sk, &rt->u.dst) && !ipfragok)
371 iph->frag_off = htons(IP_DF);
374 iph->ttl = ip_select_ttl(inet, &rt->u.dst);
375 iph->protocol = sk->sk_protocol;
376 iph->saddr = rt->rt_src;
377 iph->daddr = rt->rt_dst;
378 /* Transport layer set skb->h.foo itself. */
380 if (opt && opt->optlen) {
381 iph->ihl += opt->optlen >> 2;
382 ip_options_build(skb, opt, inet->inet_daddr, rt, 0);
385 ip_select_ident_more(iph, &rt->u.dst, sk,
386 (skb_shinfo(skb)->gso_segs ?: 1) - 1);
388 skb->priority = sk->sk_priority;
389 skb->mark = sk->sk_mark;
391 return ip_local_out(skb);
394 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
396 return -EHOSTUNREACH;
400 static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
402 to->pkt_type = from->pkt_type;
403 to->priority = from->priority;
404 to->protocol = from->protocol;
406 skb_dst_set(to, dst_clone(skb_dst(from)));
408 to->mark = from->mark;
410 /* Copy the flags to each fragment. */
411 IPCB(to)->flags = IPCB(from)->flags;
413 #ifdef CONFIG_NET_SCHED
414 to->tc_index = from->tc_index;
417 #if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
418 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
419 to->nf_trace = from->nf_trace;
421 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
422 to->ipvs_property = from->ipvs_property;
424 skb_copy_secmark(to, from);
428 * This IP datagram is too large to be sent in one piece. Break it up into
429 * smaller pieces (each of size equal to IP header plus
430 * a block of the data of the original IP data part) that will yet fit in a
431 * single device frame, and queue such a frame for sending.
434 int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
439 struct net_device *dev;
440 struct sk_buff *skb2;
441 unsigned int mtu, hlen, left, len, ll_rs, pad;
443 __be16 not_last_frag;
444 struct rtable *rt = skb_rtable(skb);
450 * Point into the IP datagram header.
455 if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) {
456 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
457 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
458 htonl(ip_skb_dst_mtu(skb)));
464 * Setup starting values.
468 mtu = dst_mtu(&rt->u.dst) - hlen; /* Size of data space */
469 IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
471 /* When frag_list is given, use it. First, check its validity:
472 * some transformers could create wrong frag_list or break existing
473 * one, it is not prohibited. In this case fall back to copying.
475 * LATER: this step can be merged to real generation of fragments,
476 * we can switch to copy when see the first bad fragment.
478 if (skb_has_frags(skb)) {
479 struct sk_buff *frag;
480 int first_len = skb_pagelen(skb);
483 if (first_len - hlen > mtu ||
484 ((first_len - hlen) & 7) ||
485 (iph->frag_off & htons(IP_MF|IP_OFFSET)) ||
489 skb_walk_frags(skb, frag) {
490 /* Correct geometry. */
491 if (frag->len > mtu ||
492 ((frag->len & 7) && frag->next) ||
493 skb_headroom(frag) < hlen)
496 /* Partially cloned skb? */
497 if (skb_shared(frag))
503 frag->destructor = sock_wfree;
504 truesizes += frag->truesize;
508 /* Everything is OK. Generate! */
512 frag = skb_shinfo(skb)->frag_list;
513 skb_frag_list_init(skb);
514 skb->data_len = first_len - skb_headlen(skb);
515 skb->truesize -= truesizes;
516 skb->len = first_len;
517 iph->tot_len = htons(first_len);
518 iph->frag_off = htons(IP_MF);
522 /* Prepare header of the next frame,
523 * before previous one went down. */
525 frag->ip_summed = CHECKSUM_NONE;
526 skb_reset_transport_header(frag);
527 __skb_push(frag, hlen);
528 skb_reset_network_header(frag);
529 memcpy(skb_network_header(frag), iph, hlen);
531 iph->tot_len = htons(frag->len);
532 ip_copy_metadata(frag, skb);
534 ip_options_fragment(frag);
535 offset += skb->len - hlen;
536 iph->frag_off = htons(offset>>3);
537 if (frag->next != NULL)
538 iph->frag_off |= htons(IP_MF);
539 /* Ready, complete checksum */
546 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
556 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
565 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
570 left = skb->len - hlen; /* Space per frame */
571 ptr = raw + hlen; /* Where to start from */
573 /* for bridged IP traffic encapsulated inside f.e. a vlan header,
574 * we need to make room for the encapsulating header
576 pad = nf_bridge_pad(skb);
577 ll_rs = LL_RESERVED_SPACE_EXTRA(rt->u.dst.dev, pad);
581 * Fragment the datagram.
584 offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
585 not_last_frag = iph->frag_off & htons(IP_MF);
588 * Keep copying data until we run out.
593 /* IF: it doesn't fit, use 'mtu' - the data space left */
596 /* IF: we are not sending upto and including the packet end
597 then align the next start on an eight byte boundary */
605 if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) {
606 NETDEBUG(KERN_INFO "IP: frag: no memory for new fragment!\n");
612 * Set up data on packet
615 ip_copy_metadata(skb2, skb);
616 skb_reserve(skb2, ll_rs);
617 skb_put(skb2, len + hlen);
618 skb_reset_network_header(skb2);
619 skb2->transport_header = skb2->network_header + hlen;
622 * Charge the memory for the fragment to any owner
627 skb_set_owner_w(skb2, skb->sk);
630 * Copy the packet header into the new buffer.
633 skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
636 * Copy a block of the IP datagram.
638 if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
643 * Fill in the new header fields.
646 iph->frag_off = htons((offset >> 3));
648 /* ANK: dirty, but effective trick. Upgrade options only if
649 * the segment to be fragmented was THE FIRST (otherwise,
650 * options are already fixed) and make it ONCE
651 * on the initial skb, so that all the following fragments
652 * will inherit fixed options.
655 ip_options_fragment(skb);
658 * Added AC : If we are fragmenting a fragment that's not the
659 * last fragment then keep MF on each bit
661 if (left > 0 || not_last_frag)
662 iph->frag_off |= htons(IP_MF);
667 * Put this fragment into the sending queue.
669 iph->tot_len = htons(len + hlen);
677 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
680 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
685 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
689 EXPORT_SYMBOL(ip_fragment);
692 ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
694 struct iovec *iov = from;
696 if (skb->ip_summed == CHECKSUM_PARTIAL) {
697 if (memcpy_fromiovecend(to, iov, offset, len) < 0)
701 if (csum_partial_copy_fromiovecend(to, iov, offset, len, &csum) < 0)
703 skb->csum = csum_block_add(skb->csum, csum, odd);
709 csum_page(struct page *page, int offset, int copy)
714 csum = csum_partial(kaddr + offset, copy, 0);
719 static inline int ip_ufo_append_data(struct sock *sk,
720 int getfrag(void *from, char *to, int offset, int len,
721 int odd, struct sk_buff *skb),
722 void *from, int length, int hh_len, int fragheaderlen,
723 int transhdrlen, int mtu, unsigned int flags)
728 /* There is support for UDP fragmentation offload by network
729 * device, so create one single skb packet containing complete
732 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL) {
733 skb = sock_alloc_send_skb(sk,
734 hh_len + fragheaderlen + transhdrlen + 20,
735 (flags & MSG_DONTWAIT), &err);
740 /* reserve space for Hardware header */
741 skb_reserve(skb, hh_len);
743 /* create space for UDP/IP header */
744 skb_put(skb, fragheaderlen + transhdrlen);
746 /* initialize network header pointer */
747 skb_reset_network_header(skb);
749 /* initialize protocol header pointer */
750 skb->transport_header = skb->network_header + fragheaderlen;
752 skb->ip_summed = CHECKSUM_PARTIAL;
754 sk->sk_sndmsg_off = 0;
756 /* specify the length of each IP datagram fragment */
757 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
758 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
759 __skb_queue_tail(&sk->sk_write_queue, skb);
762 return skb_append_datato_frags(sk, skb, getfrag, from,
763 (length - transhdrlen));
767 * ip_append_data() and ip_append_page() can make one large IP datagram
768 * from many pieces of data. Each pieces will be holded on the socket
769 * until ip_push_pending_frames() is called. Each piece can be a page
772 * Not only UDP, other transport protocols - e.g. raw sockets - can use
773 * this interface potentially.
775 * LATER: length must be adjusted by pad at tail, when it is required.
777 int ip_append_data(struct sock *sk,
778 int getfrag(void *from, char *to, int offset, int len,
779 int odd, struct sk_buff *skb),
780 void *from, int length, int transhdrlen,
781 struct ipcm_cookie *ipc, struct rtable **rtp,
784 struct inet_sock *inet = inet_sk(sk);
787 struct ip_options *opt = NULL;
794 unsigned int maxfraglen, fragheaderlen;
795 int csummode = CHECKSUM_NONE;
801 if (skb_queue_empty(&sk->sk_write_queue)) {
807 if (inet->cork.opt == NULL) {
808 inet->cork.opt = kmalloc(sizeof(struct ip_options) + 40, sk->sk_allocation);
809 if (unlikely(inet->cork.opt == NULL))
812 memcpy(inet->cork.opt, opt, sizeof(struct ip_options)+opt->optlen);
813 inet->cork.flags |= IPCORK_OPT;
814 inet->cork.addr = ipc->addr;
820 * We steal reference to this route, caller should not release it
823 inet->cork.fragsize = mtu = inet->pmtudisc == IP_PMTUDISC_PROBE ?
825 dst_mtu(rt->u.dst.path);
826 inet->cork.dst = &rt->u.dst;
827 inet->cork.length = 0;
828 sk->sk_sndmsg_page = NULL;
829 sk->sk_sndmsg_off = 0;
830 if ((exthdrlen = rt->u.dst.header_len) != 0) {
832 transhdrlen += exthdrlen;
835 rt = (struct rtable *)inet->cork.dst;
836 if (inet->cork.flags & IPCORK_OPT)
837 opt = inet->cork.opt;
841 mtu = inet->cork.fragsize;
843 hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
845 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
846 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
848 if (inet->cork.length + length > 0xFFFF - fragheaderlen) {
849 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->inet_dport,
855 * transhdrlen > 0 means that this is the first fragment and we wish
856 * it won't be fragmented in the future.
859 length + fragheaderlen <= mtu &&
860 rt->u.dst.dev->features & NETIF_F_V4_CSUM &&
862 csummode = CHECKSUM_PARTIAL;
864 inet->cork.length += length;
865 if (((length> mtu) || !skb_queue_empty(&sk->sk_write_queue)) &&
866 (sk->sk_protocol == IPPROTO_UDP) &&
867 (rt->u.dst.dev->features & NETIF_F_UFO)) {
868 err = ip_ufo_append_data(sk, getfrag, from, length, hh_len,
869 fragheaderlen, transhdrlen, mtu,
876 /* So, what's going on in the loop below?
878 * We use calculated fragment length to generate chained skb,
879 * each of segments is IP fragment ready for sending to network after
880 * adding appropriate IP header.
883 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
887 /* Check if the remaining data fits into current packet. */
888 copy = mtu - skb->len;
890 copy = maxfraglen - skb->len;
893 unsigned int datalen;
894 unsigned int fraglen;
895 unsigned int fraggap;
896 unsigned int alloclen;
897 struct sk_buff *skb_prev;
901 fraggap = skb_prev->len - maxfraglen;
906 * If remaining data exceeds the mtu,
907 * we know we need more fragment(s).
909 datalen = length + fraggap;
910 if (datalen > mtu - fragheaderlen)
911 datalen = maxfraglen - fragheaderlen;
912 fraglen = datalen + fragheaderlen;
914 if ((flags & MSG_MORE) &&
915 !(rt->u.dst.dev->features&NETIF_F_SG))
918 alloclen = datalen + fragheaderlen;
920 /* The last fragment gets additional space at tail.
921 * Note, with MSG_MORE we overallocate on fragments,
922 * because we have no idea what fragment will be
925 if (datalen == length + fraggap)
926 alloclen += rt->u.dst.trailer_len;
929 skb = sock_alloc_send_skb(sk,
930 alloclen + hh_len + 15,
931 (flags & MSG_DONTWAIT), &err);
934 if (atomic_read(&sk->sk_wmem_alloc) <=
936 skb = sock_wmalloc(sk,
937 alloclen + hh_len + 15, 1,
939 if (unlikely(skb == NULL))
942 /* only the initial fragment is
950 * Fill in the control structures
952 skb->ip_summed = csummode;
954 skb_reserve(skb, hh_len);
955 *skb_tx(skb) = ipc->shtx;
958 * Find where to start putting bytes.
960 data = skb_put(skb, fraglen);
961 skb_set_network_header(skb, exthdrlen);
962 skb->transport_header = (skb->network_header +
964 data += fragheaderlen;
967 skb->csum = skb_copy_and_csum_bits(
968 skb_prev, maxfraglen,
969 data + transhdrlen, fraggap, 0);
970 skb_prev->csum = csum_sub(skb_prev->csum,
973 pskb_trim_unique(skb_prev, maxfraglen);
976 copy = datalen - transhdrlen - fraggap;
977 if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
984 length -= datalen - fraggap;
987 csummode = CHECKSUM_NONE;
990 * Put the packet on the pending queue.
992 __skb_queue_tail(&sk->sk_write_queue, skb);
999 if (!(rt->u.dst.dev->features&NETIF_F_SG)) {
1003 if (getfrag(from, skb_put(skb, copy),
1004 offset, copy, off, skb) < 0) {
1005 __skb_trim(skb, off);
1010 int i = skb_shinfo(skb)->nr_frags;
1011 skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
1012 struct page *page = sk->sk_sndmsg_page;
1013 int off = sk->sk_sndmsg_off;
1016 if (page && (left = PAGE_SIZE - off) > 0) {
1019 if (page != frag->page) {
1020 if (i == MAX_SKB_FRAGS) {
1025 skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0);
1026 frag = &skb_shinfo(skb)->frags[i];
1028 } else if (i < MAX_SKB_FRAGS) {
1029 if (copy > PAGE_SIZE)
1031 page = alloc_pages(sk->sk_allocation, 0);
1036 sk->sk_sndmsg_page = page;
1037 sk->sk_sndmsg_off = 0;
1039 skb_fill_page_desc(skb, i, page, 0, 0);
1040 frag = &skb_shinfo(skb)->frags[i];
1045 if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) {
1049 sk->sk_sndmsg_off += copy;
1052 skb->data_len += copy;
1053 skb->truesize += copy;
1054 atomic_add(copy, &sk->sk_wmem_alloc);
1063 inet->cork.length -= length;
1064 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
1068 ssize_t ip_append_page(struct sock *sk, struct page *page,
1069 int offset, size_t size, int flags)
1071 struct inet_sock *inet = inet_sk(sk);
1072 struct sk_buff *skb;
1074 struct ip_options *opt = NULL;
1079 unsigned int maxfraglen, fragheaderlen, fraggap;
1084 if (flags&MSG_PROBE)
1087 if (skb_queue_empty(&sk->sk_write_queue))
1090 rt = (struct rtable *)inet->cork.dst;
1091 if (inet->cork.flags & IPCORK_OPT)
1092 opt = inet->cork.opt;
1094 if (!(rt->u.dst.dev->features&NETIF_F_SG))
1097 hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
1098 mtu = inet->cork.fragsize;
1100 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
1101 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
1103 if (inet->cork.length + size > 0xFFFF - fragheaderlen) {
1104 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->inet_dport, mtu);
1108 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
1111 inet->cork.length += size;
1112 if ((sk->sk_protocol == IPPROTO_UDP) &&
1113 (rt->u.dst.dev->features & NETIF_F_UFO)) {
1114 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
1115 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1122 if (skb_is_gso(skb))
1126 /* Check if the remaining data fits into current packet. */
1127 len = mtu - skb->len;
1129 len = maxfraglen - skb->len;
1132 struct sk_buff *skb_prev;
1136 fraggap = skb_prev->len - maxfraglen;
1138 alloclen = fragheaderlen + hh_len + fraggap + 15;
1139 skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
1140 if (unlikely(!skb)) {
1146 * Fill in the control structures
1148 skb->ip_summed = CHECKSUM_NONE;
1150 skb_reserve(skb, hh_len);
1153 * Find where to start putting bytes.
1155 skb_put(skb, fragheaderlen + fraggap);
1156 skb_reset_network_header(skb);
1157 skb->transport_header = (skb->network_header +
1160 skb->csum = skb_copy_and_csum_bits(skb_prev,
1162 skb_transport_header(skb),
1164 skb_prev->csum = csum_sub(skb_prev->csum,
1166 pskb_trim_unique(skb_prev, maxfraglen);
1170 * Put the packet on the pending queue.
1172 __skb_queue_tail(&sk->sk_write_queue, skb);
1176 i = skb_shinfo(skb)->nr_frags;
1179 if (skb_can_coalesce(skb, i, page, offset)) {
1180 skb_shinfo(skb)->frags[i-1].size += len;
1181 } else if (i < MAX_SKB_FRAGS) {
1183 skb_fill_page_desc(skb, i, page, offset, len);
1189 if (skb->ip_summed == CHECKSUM_NONE) {
1191 csum = csum_page(page, offset, len);
1192 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1196 skb->data_len += len;
1197 skb->truesize += len;
1198 atomic_add(len, &sk->sk_wmem_alloc);
1205 inet->cork.length -= size;
1206 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
1210 static void ip_cork_release(struct inet_sock *inet)
1212 inet->cork.flags &= ~IPCORK_OPT;
1213 kfree(inet->cork.opt);
1214 inet->cork.opt = NULL;
1215 dst_release(inet->cork.dst);
1216 inet->cork.dst = NULL;
1220 * Combined all pending IP fragments on the socket as one IP datagram
1221 * and push them out.
1223 int ip_push_pending_frames(struct sock *sk)
1225 struct sk_buff *skb, *tmp_skb;
1226 struct sk_buff **tail_skb;
1227 struct inet_sock *inet = inet_sk(sk);
1228 struct net *net = sock_net(sk);
1229 struct ip_options *opt = NULL;
1230 struct rtable *rt = (struct rtable *)inet->cork.dst;
1236 if ((skb = __skb_dequeue(&sk->sk_write_queue)) == NULL)
1238 tail_skb = &(skb_shinfo(skb)->frag_list);
1240 /* move skb->data to ip header from ext header */
1241 if (skb->data < skb_network_header(skb))
1242 __skb_pull(skb, skb_network_offset(skb));
1243 while ((tmp_skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
1244 __skb_pull(tmp_skb, skb_network_header_len(skb));
1245 *tail_skb = tmp_skb;
1246 tail_skb = &(tmp_skb->next);
1247 skb->len += tmp_skb->len;
1248 skb->data_len += tmp_skb->len;
1249 skb->truesize += tmp_skb->truesize;
1250 tmp_skb->destructor = NULL;
1254 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1255 * to fragment the frame generated here. No matter, what transforms
1256 * how transforms change size of the packet, it will come out.
1258 if (inet->pmtudisc < IP_PMTUDISC_DO)
1261 /* DF bit is set when we want to see DF on outgoing frames.
1262 * If local_df is set too, we still allow to fragment this frame
1264 if (inet->pmtudisc >= IP_PMTUDISC_DO ||
1265 (skb->len <= dst_mtu(&rt->u.dst) &&
1266 ip_dont_fragment(sk, &rt->u.dst)))
1269 if (inet->cork.flags & IPCORK_OPT)
1270 opt = inet->cork.opt;
1272 if (rt->rt_type == RTN_MULTICAST)
1275 ttl = ip_select_ttl(inet, &rt->u.dst);
1277 iph = (struct iphdr *)skb->data;
1281 iph->ihl += opt->optlen>>2;
1282 ip_options_build(skb, opt, inet->cork.addr, rt, 0);
1284 iph->tos = inet->tos;
1286 ip_select_ident(iph, &rt->u.dst, sk);
1288 iph->protocol = sk->sk_protocol;
1289 iph->saddr = rt->rt_src;
1290 iph->daddr = rt->rt_dst;
1292 skb->priority = sk->sk_priority;
1293 skb->mark = sk->sk_mark;
1295 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
1298 inet->cork.dst = NULL;
1299 skb_dst_set(skb, &rt->u.dst);
1301 if (iph->protocol == IPPROTO_ICMP)
1302 icmp_out_count(net, ((struct icmphdr *)
1303 skb_transport_header(skb))->type);
1305 /* Netfilter gets whole the not fragmented skb. */
1306 err = ip_local_out(skb);
1309 err = net_xmit_errno(err);
1315 ip_cork_release(inet);
1319 IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
1324 * Throw away all pending data on the socket.
1326 void ip_flush_pending_frames(struct sock *sk)
1328 struct sk_buff *skb;
1330 while ((skb = __skb_dequeue_tail(&sk->sk_write_queue)) != NULL)
1333 ip_cork_release(inet_sk(sk));
1338 * Fetch data from kernel space and fill in checksum if needed.
1340 static int ip_reply_glue_bits(void *dptr, char *to, int offset,
1341 int len, int odd, struct sk_buff *skb)
1345 csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
1346 skb->csum = csum_block_add(skb->csum, csum, odd);
1351 * Generic function to send a packet as reply to another packet.
1352 * Used to send TCP resets so far. ICMP should use this function too.
1354 * Should run single threaded per socket because it uses the sock
1355 * structure to pass arguments.
1357 void ip_send_reply(struct sock *sk, struct sk_buff *skb, struct ip_reply_arg *arg,
1360 struct inet_sock *inet = inet_sk(sk);
1362 struct ip_options opt;
1365 struct ipcm_cookie ipc;
1367 struct rtable *rt = skb_rtable(skb);
1369 if (ip_options_echo(&replyopts.opt, skb))
1372 daddr = ipc.addr = rt->rt_src;
1376 if (replyopts.opt.optlen) {
1377 ipc.opt = &replyopts.opt;
1380 daddr = replyopts.opt.faddr;
1384 struct flowi fl = { .oif = arg->bound_dev_if,
1387 .saddr = rt->rt_spec_dst,
1388 .tos = RT_TOS(ip_hdr(skb)->tos) } },
1389 /* Not quite clean, but right. */
1391 { .sport = tcp_hdr(skb)->dest,
1392 .dport = tcp_hdr(skb)->source } },
1393 .proto = sk->sk_protocol,
1394 .flags = ip_reply_arg_flowi_flags(arg) };
1395 security_skb_classify_flow(skb, &fl);
1396 if (ip_route_output_key(sock_net(sk), &rt, &fl))
1400 /* And let IP do all the hard work.
1402 This chunk is not reenterable, hence spinlock.
1403 Note that it uses the fact, that this function is called
1404 with locally disabled BH and that sk cannot be already spinlocked.
1407 inet->tos = ip_hdr(skb)->tos;
1408 sk->sk_priority = skb->priority;
1409 sk->sk_protocol = ip_hdr(skb)->protocol;
1410 sk->sk_bound_dev_if = arg->bound_dev_if;
1411 ip_append_data(sk, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
1412 &ipc, &rt, MSG_DONTWAIT);
1413 if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
1414 if (arg->csumoffset >= 0)
1415 *((__sum16 *)skb_transport_header(skb) +
1416 arg->csumoffset) = csum_fold(csum_add(skb->csum,
1418 skb->ip_summed = CHECKSUM_NONE;
1419 ip_push_pending_frames(sk);
1427 void __init ip_init(void)
1432 #if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1433 igmp_mc_proc_init();
1437 EXPORT_SYMBOL(ip_generic_getfrag);
1438 EXPORT_SYMBOL(ip_queue_xmit);
1439 EXPORT_SYMBOL(ip_send_check);