2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * The Internet Protocol (IP) output module.
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Donald Becker, <becker@super.org>
11 * Alan Cox, <Alan.Cox@linux.org>
13 * Stefan Becker, <stefanb@yello.ping.de>
14 * Jorge Cwik, <jorge@laser.satlink.net>
15 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
16 * Hirokazu Takahashi, <taka@valinux.co.jp>
18 * See ip_input.c for original log
21 * Alan Cox : Missing nonblock feature in ip_build_xmit.
22 * Mike Kilburn : htons() missing in ip_build_xmit.
23 * Bradford Johnson: Fix faulty handling of some frames when
25 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit
26 * (in case if packet not accepted by
27 * output firewall rules)
28 * Mike McLagan : Routing by source
29 * Alexey Kuznetsov: use new route cache
30 * Andi Kleen: Fix broken PMTU recovery and remove
31 * some redundant tests.
32 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
33 * Andi Kleen : Replace ip_reply with ip_send_reply.
34 * Andi Kleen : Split fast and slow ip_build_xmit path
35 * for decreased register pressure on x86
36 * and more readibility.
37 * Marc Boucher : When call_out_firewall returns FW_QUEUE,
38 * silently drop skb instead of failing with -EPERM.
39 * Detlev Wengorz : Copy protocol for fragments.
40 * Hirokazu Takahashi: HW checksumming for outgoing UDP
42 * Hirokazu Takahashi: sendfile() on UDP works now.
45 #include <asm/uaccess.h>
46 #include <linux/module.h>
47 #include <linux/types.h>
48 #include <linux/kernel.h>
50 #include <linux/string.h>
51 #include <linux/errno.h>
52 #include <linux/highmem.h>
53 #include <linux/slab.h>
55 #include <linux/socket.h>
56 #include <linux/sockios.h>
58 #include <linux/inet.h>
59 #include <linux/netdevice.h>
60 #include <linux/etherdevice.h>
61 #include <linux/proc_fs.h>
62 #include <linux/stat.h>
63 #include <linux/init.h>
67 #include <net/protocol.h>
68 #include <net/route.h>
70 #include <linux/skbuff.h>
74 #include <net/checksum.h>
75 #include <net/inetpeer.h>
76 #include <linux/igmp.h>
77 #include <linux/netfilter_ipv4.h>
78 #include <linux/netfilter_bridge.h>
79 #include <linux/mroute.h>
80 #include <linux/netlink.h>
81 #include <linux/tcp.h>
83 int sysctl_ip_default_ttl __read_mostly = IPDEFTTL;
84 EXPORT_SYMBOL(sysctl_ip_default_ttl);
86 /* Generate a checksum for an outgoing IP datagram. */
87 __inline__ void ip_send_check(struct iphdr *iph)
90 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
92 EXPORT_SYMBOL(ip_send_check);
94 int __ip_local_out(struct sk_buff *skb)
96 struct iphdr *iph = ip_hdr(skb);
98 iph->tot_len = htons(skb->len);
100 return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, skb, NULL,
101 skb_dst(skb)->dev, dst_output);
104 int ip_local_out(struct sk_buff *skb)
108 err = __ip_local_out(skb);
109 if (likely(err == 1))
110 err = dst_output(skb);
114 EXPORT_SYMBOL_GPL(ip_local_out);
116 /* dev_loopback_xmit for use with netfilter. */
117 static int ip_dev_loopback_xmit(struct sk_buff *newskb)
119 skb_reset_mac_header(newskb);
120 __skb_pull(newskb, skb_network_offset(newskb));
121 newskb->pkt_type = PACKET_LOOPBACK;
122 newskb->ip_summed = CHECKSUM_UNNECESSARY;
123 WARN_ON(!skb_dst(newskb));
124 skb_dst_force(newskb);
129 static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
131 int ttl = inet->uc_ttl;
134 ttl = ip4_dst_hoplimit(dst);
139 * Add an ip header to a skbuff and send it out.
142 int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk,
143 __be32 saddr, __be32 daddr, struct ip_options_rcu *opt)
145 struct inet_sock *inet = inet_sk(sk);
146 struct rtable *rt = skb_rtable(skb);
149 /* Build the IP header. */
150 skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0));
151 skb_reset_network_header(skb);
155 iph->tos = inet->tos;
156 if (ip_dont_fragment(sk, &rt->dst))
157 iph->frag_off = htons(IP_DF);
160 iph->ttl = ip_select_ttl(inet, &rt->dst);
161 iph->daddr = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
163 iph->protocol = sk->sk_protocol;
164 ip_select_ident(skb, sk);
166 if (opt && opt->opt.optlen) {
167 iph->ihl += opt->opt.optlen>>2;
168 ip_options_build(skb, &opt->opt, daddr, rt, 0);
171 skb->priority = sk->sk_priority;
172 skb->mark = sk->sk_mark;
175 return ip_local_out(skb);
177 EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);
179 static inline int ip_finish_output2(struct sk_buff *skb)
181 struct dst_entry *dst = skb_dst(skb);
182 struct rtable *rt = (struct rtable *)dst;
183 struct net_device *dev = dst->dev;
184 unsigned int hh_len = LL_RESERVED_SPACE(dev);
185 struct neighbour *neigh;
187 if (rt->rt_type == RTN_MULTICAST) {
188 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len);
189 } else if (rt->rt_type == RTN_BROADCAST)
190 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTBCAST, skb->len);
192 /* Be paranoid, rather than too clever. */
193 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
194 struct sk_buff *skb2;
196 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
202 skb_set_owner_w(skb2, skb->sk);
208 neigh = dst_get_neighbour_noref(dst);
210 int res = neigh_output(neigh, skb);
218 printk(KERN_DEBUG "ip_finish_output2: No header cache and no neighbour!\n");
223 static inline int ip_skb_dst_mtu(struct sk_buff *skb)
225 struct inet_sock *inet = skb->sk ? inet_sk(skb->sk) : NULL;
227 return (inet && inet->pmtudisc == IP_PMTUDISC_PROBE) ?
228 skb_dst(skb)->dev->mtu : dst_mtu(skb_dst(skb));
231 static int ip_finish_output(struct sk_buff *skb)
233 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
234 /* Policy lookup after SNAT yielded a new policy */
235 if (skb_dst(skb)->xfrm != NULL) {
236 IPCB(skb)->flags |= IPSKB_REROUTED;
237 return dst_output(skb);
240 if (skb->len > ip_skb_dst_mtu(skb) && !skb_is_gso(skb))
241 return ip_fragment(skb, ip_finish_output2);
243 return ip_finish_output2(skb);
246 int ip_mc_output(struct sk_buff *skb)
248 struct sock *sk = skb->sk;
249 struct rtable *rt = skb_rtable(skb);
250 struct net_device *dev = rt->dst.dev;
253 * If the indicated interface is up and running, send the packet.
255 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
258 skb->protocol = htons(ETH_P_IP);
261 * Multicasts are looped back for other local users
264 if (rt->rt_flags&RTCF_MULTICAST) {
266 #ifdef CONFIG_IP_MROUTE
267 /* Small optimization: do not loopback not local frames,
268 which returned after forwarding; they will be dropped
269 by ip_mr_input in any case.
270 Note, that local frames are looped back to be delivered
273 This check is duplicated in ip_mr_input at the moment.
276 ((rt->rt_flags & RTCF_LOCAL) ||
277 !(IPCB(skb)->flags & IPSKB_FORWARDED))
280 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
282 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
283 newskb, NULL, newskb->dev,
284 ip_dev_loopback_xmit);
287 /* Multicasts with ttl 0 must not go beyond the host */
289 if (ip_hdr(skb)->ttl == 0) {
295 if (rt->rt_flags&RTCF_BROADCAST) {
296 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
298 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, newskb,
299 NULL, newskb->dev, ip_dev_loopback_xmit);
302 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL,
303 skb->dev, ip_finish_output,
304 !(IPCB(skb)->flags & IPSKB_REROUTED));
307 int ip_output(struct sk_buff *skb)
309 struct net_device *dev = skb_dst(skb)->dev;
311 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
314 skb->protocol = htons(ETH_P_IP);
316 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL, dev,
318 !(IPCB(skb)->flags & IPSKB_REROUTED));
322 * copy saddr and daddr, possibly using 64bit load/stores
324 * iph->saddr = fl4->saddr;
325 * iph->daddr = fl4->daddr;
327 static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4)
329 BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) !=
330 offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr));
331 memcpy(&iph->saddr, &fl4->saddr,
332 sizeof(fl4->saddr) + sizeof(fl4->daddr));
335 int ip_queue_xmit(struct sk_buff *skb, struct flowi *fl)
337 struct sock *sk = skb->sk;
338 struct inet_sock *inet = inet_sk(sk);
339 struct ip_options_rcu *inet_opt;
345 /* Skip all of this if the packet is already routed,
346 * f.e. by something like SCTP.
349 inet_opt = rcu_dereference(inet->inet_opt);
351 rt = skb_rtable(skb);
355 /* Make sure we can route this packet. */
356 rt = (struct rtable *)__sk_dst_check(sk, 0);
360 /* Use correct destination address if we have options. */
361 daddr = inet->inet_daddr;
362 if (inet_opt && inet_opt->opt.srr)
363 daddr = inet_opt->opt.faddr;
365 /* If this fails, retransmit mechanism of transport layer will
366 * keep trying until route appears or the connection times
369 rt = ip_route_output_ports(sock_net(sk), fl4, sk,
370 daddr, inet->inet_saddr,
375 sk->sk_bound_dev_if);
378 sk_setup_caps(sk, &rt->dst);
380 skb_dst_set_noref(skb, &rt->dst);
383 if (inet_opt && inet_opt->opt.is_strictroute && fl4->daddr != rt->rt_gateway)
386 /* OK, we know where to send it, allocate and build IP header. */
387 skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0));
388 skb_reset_network_header(skb);
390 *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
391 if (ip_dont_fragment(sk, &rt->dst) && !skb->local_df)
392 iph->frag_off = htons(IP_DF);
395 iph->ttl = ip_select_ttl(inet, &rt->dst);
396 iph->protocol = sk->sk_protocol;
397 ip_copy_addrs(iph, fl4);
399 /* Transport layer set skb->h.foo itself. */
401 if (inet_opt && inet_opt->opt.optlen) {
402 iph->ihl += inet_opt->opt.optlen >> 2;
403 ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0);
406 ip_select_ident_segs(skb, sk, skb_shinfo(skb)->gso_segs ?: 1);
408 skb->priority = sk->sk_priority;
409 skb->mark = sk->sk_mark;
411 res = ip_local_out(skb);
417 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
419 return -EHOSTUNREACH;
421 EXPORT_SYMBOL(ip_queue_xmit);
424 static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
426 to->pkt_type = from->pkt_type;
427 to->priority = from->priority;
428 to->protocol = from->protocol;
430 skb_dst_copy(to, from);
432 to->mark = from->mark;
434 /* Copy the flags to each fragment. */
435 IPCB(to)->flags = IPCB(from)->flags;
437 #ifdef CONFIG_NET_SCHED
438 to->tc_index = from->tc_index;
441 #if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
442 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
443 to->nf_trace = from->nf_trace;
445 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
446 to->ipvs_property = from->ipvs_property;
448 skb_copy_secmark(to, from);
452 * This IP datagram is too large to be sent in one piece. Break it up into
453 * smaller pieces (each of size equal to IP header plus
454 * a block of the data of the original IP data part) that will yet fit in a
455 * single device frame, and queue such a frame for sending.
458 int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
462 struct net_device *dev;
463 struct sk_buff *skb2;
464 unsigned int mtu, hlen, left, len, ll_rs;
466 __be16 not_last_frag;
467 struct rtable *rt = skb_rtable(skb);
473 * Point into the IP datagram header.
478 if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) {
479 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
480 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
481 htonl(ip_skb_dst_mtu(skb)));
487 * Setup starting values.
491 mtu = dst_mtu(&rt->dst) - hlen; /* Size of data space */
492 #ifdef CONFIG_BRIDGE_NETFILTER
494 mtu -= nf_bridge_mtu_reduction(skb);
496 IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
498 /* When frag_list is given, use it. First, check its validity:
499 * some transformers could create wrong frag_list or break existing
500 * one, it is not prohibited. In this case fall back to copying.
502 * LATER: this step can be merged to real generation of fragments,
503 * we can switch to copy when see the first bad fragment.
505 if (skb_has_frag_list(skb)) {
506 struct sk_buff *frag, *frag2;
507 int first_len = skb_pagelen(skb);
509 if (first_len - hlen > mtu ||
510 ((first_len - hlen) & 7) ||
511 ip_is_fragment(iph) ||
515 skb_walk_frags(skb, frag) {
516 /* Correct geometry. */
517 if (frag->len > mtu ||
518 ((frag->len & 7) && frag->next) ||
519 skb_headroom(frag) < hlen)
520 goto slow_path_clean;
522 /* Partially cloned skb? */
523 if (skb_shared(frag))
524 goto slow_path_clean;
529 frag->destructor = sock_wfree;
531 skb->truesize -= frag->truesize;
534 /* Everything is OK. Generate! */
538 frag = skb_shinfo(skb)->frag_list;
539 skb_frag_list_init(skb);
540 skb->data_len = first_len - skb_headlen(skb);
541 skb->len = first_len;
542 iph->tot_len = htons(first_len);
543 iph->frag_off = htons(IP_MF);
547 /* Prepare header of the next frame,
548 * before previous one went down. */
550 frag->ip_summed = CHECKSUM_NONE;
551 skb_reset_transport_header(frag);
552 __skb_push(frag, hlen);
553 skb_reset_network_header(frag);
554 memcpy(skb_network_header(frag), iph, hlen);
556 iph->tot_len = htons(frag->len);
557 ip_copy_metadata(frag, skb);
559 ip_options_fragment(frag);
560 offset += skb->len - hlen;
561 iph->frag_off = htons(offset>>3);
562 if (frag->next != NULL)
563 iph->frag_off |= htons(IP_MF);
564 /* Ready, complete checksum */
571 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
581 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
590 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
594 skb_walk_frags(skb, frag2) {
598 frag2->destructor = NULL;
599 skb->truesize += frag2->truesize;
604 left = skb->len - hlen; /* Space per frame */
605 ptr = hlen; /* Where to start from */
607 /* for bridged IP traffic encapsulated inside f.e. a vlan header,
608 * we need to make room for the encapsulating header
610 ll_rs = LL_RESERVED_SPACE_EXTRA(rt->dst.dev, nf_bridge_pad(skb));
613 * Fragment the datagram.
616 offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
617 not_last_frag = iph->frag_off & htons(IP_MF);
620 * Keep copying data until we run out.
625 /* IF: it doesn't fit, use 'mtu' - the data space left */
628 /* IF: we are not sending up to and including the packet end
629 then align the next start on an eight byte boundary */
637 if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) {
638 NETDEBUG(KERN_INFO "IP: frag: no memory for new fragment!\n");
644 * Set up data on packet
647 ip_copy_metadata(skb2, skb);
648 skb_reserve(skb2, ll_rs);
649 skb_put(skb2, len + hlen);
650 skb_reset_network_header(skb2);
651 skb2->transport_header = skb2->network_header + hlen;
654 * Charge the memory for the fragment to any owner
659 skb_set_owner_w(skb2, skb->sk);
662 * Copy the packet header into the new buffer.
665 skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
668 * Copy a block of the IP datagram.
670 if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
675 * Fill in the new header fields.
678 iph->frag_off = htons((offset >> 3));
680 /* ANK: dirty, but effective trick. Upgrade options only if
681 * the segment to be fragmented was THE FIRST (otherwise,
682 * options are already fixed) and make it ONCE
683 * on the initial skb, so that all the following fragments
684 * will inherit fixed options.
687 ip_options_fragment(skb);
690 * Added AC : If we are fragmenting a fragment that's not the
691 * last fragment then keep MF on each bit
693 if (left > 0 || not_last_frag)
694 iph->frag_off |= htons(IP_MF);
699 * Put this fragment into the sending queue.
701 iph->tot_len = htons(len + hlen);
709 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
712 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
717 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
720 EXPORT_SYMBOL(ip_fragment);
723 ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
725 struct iovec *iov = from;
727 if (skb->ip_summed == CHECKSUM_PARTIAL) {
728 if (memcpy_fromiovecend(to, iov, offset, len) < 0)
732 if (csum_partial_copy_fromiovecend(to, iov, offset, len, &csum) < 0)
734 skb->csum = csum_block_add(skb->csum, csum, odd);
738 EXPORT_SYMBOL(ip_generic_getfrag);
741 csum_page(struct page *page, int offset, int copy)
746 csum = csum_partial(kaddr + offset, copy, 0);
751 static inline int ip_ufo_append_data(struct sock *sk,
752 struct sk_buff_head *queue,
753 int getfrag(void *from, char *to, int offset, int len,
754 int odd, struct sk_buff *skb),
755 void *from, int length, int hh_len, int fragheaderlen,
756 int transhdrlen, int maxfraglen, unsigned int flags)
761 /* There is support for UDP fragmentation offload by network
762 * device, so create one single skb packet containing complete
765 if ((skb = skb_peek_tail(queue)) == NULL) {
766 skb = sock_alloc_send_skb(sk,
767 hh_len + fragheaderlen + transhdrlen + 20,
768 (flags & MSG_DONTWAIT), &err);
773 /* reserve space for Hardware header */
774 skb_reserve(skb, hh_len);
776 /* create space for UDP/IP header */
777 skb_put(skb, fragheaderlen + transhdrlen);
779 /* initialize network header pointer */
780 skb_reset_network_header(skb);
782 /* initialize protocol header pointer */
783 skb->transport_header = skb->network_header + fragheaderlen;
785 skb->ip_summed = CHECKSUM_PARTIAL;
788 /* specify the length of each IP datagram fragment */
789 skb_shinfo(skb)->gso_size = maxfraglen - fragheaderlen;
790 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
791 __skb_queue_tail(queue, skb);
794 return skb_append_datato_frags(sk, skb, getfrag, from,
795 (length - transhdrlen));
798 static int __ip_append_data(struct sock *sk,
800 struct sk_buff_head *queue,
801 struct inet_cork *cork,
802 int getfrag(void *from, char *to, int offset,
803 int len, int odd, struct sk_buff *skb),
804 void *from, int length, int transhdrlen,
807 struct inet_sock *inet = inet_sk(sk);
810 struct ip_options *opt = cork->opt;
817 unsigned int maxfraglen, fragheaderlen;
818 int csummode = CHECKSUM_NONE;
819 struct rtable *rt = (struct rtable *)cork->dst;
821 skb = skb_peek_tail(queue);
823 exthdrlen = !skb ? rt->dst.header_len : 0;
824 mtu = cork->fragsize;
826 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
828 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
829 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
831 if (cork->length + length > 0xFFFF - fragheaderlen) {
832 ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
838 * transhdrlen > 0 means that this is the first fragment and we wish
839 * it won't be fragmented in the future.
842 length + fragheaderlen <= mtu &&
843 rt->dst.dev->features & NETIF_F_V4_CSUM &&
845 csummode = CHECKSUM_PARTIAL;
847 cork->length += length;
848 if (((length > mtu) || (skb && skb_has_frags(skb))) &&
849 (sk->sk_protocol == IPPROTO_UDP) &&
850 (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len) {
851 err = ip_ufo_append_data(sk, queue, getfrag, from, length,
852 hh_len, fragheaderlen, transhdrlen,
859 /* So, what's going on in the loop below?
861 * We use calculated fragment length to generate chained skb,
862 * each of segments is IP fragment ready for sending to network after
863 * adding appropriate IP header.
870 /* Check if the remaining data fits into current packet. */
871 copy = mtu - skb->len;
873 copy = maxfraglen - skb->len;
876 unsigned int datalen;
877 unsigned int fraglen;
878 unsigned int fraggap;
879 unsigned int alloclen;
880 struct sk_buff *skb_prev;
884 fraggap = skb_prev->len - maxfraglen;
889 * If remaining data exceeds the mtu,
890 * we know we need more fragment(s).
892 datalen = length + fraggap;
893 if (datalen > mtu - fragheaderlen)
894 datalen = maxfraglen - fragheaderlen;
895 fraglen = datalen + fragheaderlen;
897 if ((flags & MSG_MORE) &&
898 !(rt->dst.dev->features&NETIF_F_SG))
903 alloclen += exthdrlen;
905 /* The last fragment gets additional space at tail.
906 * Note, with MSG_MORE we overallocate on fragments,
907 * because we have no idea what fragment will be
910 if (datalen == length + fraggap)
911 alloclen += rt->dst.trailer_len;
914 skb = sock_alloc_send_skb(sk,
915 alloclen + hh_len + 15,
916 (flags & MSG_DONTWAIT), &err);
919 if (atomic_read(&sk->sk_wmem_alloc) <=
921 skb = sock_wmalloc(sk,
922 alloclen + hh_len + 15, 1,
924 if (unlikely(skb == NULL))
927 /* only the initial fragment is
935 * Fill in the control structures
937 skb->ip_summed = csummode;
939 skb_reserve(skb, hh_len);
940 skb_shinfo(skb)->tx_flags = cork->tx_flags;
943 * Find where to start putting bytes.
945 data = skb_put(skb, fraglen + exthdrlen);
946 skb_set_network_header(skb, exthdrlen);
947 skb->transport_header = (skb->network_header +
949 data += fragheaderlen + exthdrlen;
952 skb->csum = skb_copy_and_csum_bits(
953 skb_prev, maxfraglen,
954 data + transhdrlen, fraggap, 0);
955 skb_prev->csum = csum_sub(skb_prev->csum,
958 pskb_trim_unique(skb_prev, maxfraglen);
961 copy = datalen - transhdrlen - fraggap;
962 if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
969 length -= datalen - fraggap;
972 csummode = CHECKSUM_NONE;
975 * Put the packet on the pending queue.
977 __skb_queue_tail(queue, skb);
984 if (!(rt->dst.dev->features&NETIF_F_SG)) {
988 if (getfrag(from, skb_put(skb, copy),
989 offset, copy, off, skb) < 0) {
990 __skb_trim(skb, off);
995 int i = skb_shinfo(skb)->nr_frags;
996 skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
997 struct page *page = cork->page;
1001 if (page && (left = PAGE_SIZE - off) > 0) {
1004 if (page != skb_frag_page(frag)) {
1005 if (i == MAX_SKB_FRAGS) {
1009 skb_fill_page_desc(skb, i, page, off, 0);
1010 skb_frag_ref(skb, i);
1011 frag = &skb_shinfo(skb)->frags[i];
1013 } else if (i < MAX_SKB_FRAGS) {
1014 if (copy > PAGE_SIZE)
1016 page = alloc_pages(sk->sk_allocation, 0);
1024 skb_fill_page_desc(skb, i, page, 0, 0);
1025 frag = &skb_shinfo(skb)->frags[i];
1030 if (getfrag(from, skb_frag_address(frag)+skb_frag_size(frag),
1031 offset, copy, skb->len, skb) < 0) {
1036 skb_frag_size_add(frag, copy);
1038 skb->data_len += copy;
1039 skb->truesize += copy;
1040 atomic_add(copy, &sk->sk_wmem_alloc);
1049 cork->length -= length;
1050 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
1054 static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
1055 struct ipcm_cookie *ipc, struct rtable **rtp)
1057 struct inet_sock *inet = inet_sk(sk);
1058 struct ip_options_rcu *opt;
1062 * setup for corking.
1066 if (cork->opt == NULL) {
1067 cork->opt = kmalloc(sizeof(struct ip_options) + 40,
1069 if (unlikely(cork->opt == NULL))
1072 memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen);
1073 cork->flags |= IPCORK_OPT;
1074 cork->addr = ipc->addr;
1080 * We steal reference to this route, caller should not release it
1083 cork->fragsize = inet->pmtudisc == IP_PMTUDISC_PROBE ?
1084 rt->dst.dev->mtu : dst_mtu(&rt->dst);
1085 cork->dst = &rt->dst;
1087 cork->tx_flags = ipc->tx_flags;
1095 * ip_append_data() and ip_append_page() can make one large IP datagram
1096 * from many pieces of data. Each pieces will be holded on the socket
1097 * until ip_push_pending_frames() is called. Each piece can be a page
1100 * Not only UDP, other transport protocols - e.g. raw sockets - can use
1101 * this interface potentially.
1103 * LATER: length must be adjusted by pad at tail, when it is required.
1105 int ip_append_data(struct sock *sk, struct flowi4 *fl4,
1106 int getfrag(void *from, char *to, int offset, int len,
1107 int odd, struct sk_buff *skb),
1108 void *from, int length, int transhdrlen,
1109 struct ipcm_cookie *ipc, struct rtable **rtp,
1112 struct inet_sock *inet = inet_sk(sk);
1115 if (flags&MSG_PROBE)
1118 if (skb_queue_empty(&sk->sk_write_queue)) {
1119 err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp);
1126 return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base, getfrag,
1127 from, length, transhdrlen, flags);
1130 ssize_t ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page,
1131 int offset, size_t size, int flags)
1133 struct inet_sock *inet = inet_sk(sk);
1134 struct sk_buff *skb;
1136 struct ip_options *opt = NULL;
1137 struct inet_cork *cork;
1142 unsigned int maxfraglen, fragheaderlen, fraggap;
1147 if (flags&MSG_PROBE)
1150 if (skb_queue_empty(&sk->sk_write_queue))
1153 cork = &inet->cork.base;
1154 rt = (struct rtable *)cork->dst;
1155 if (cork->flags & IPCORK_OPT)
1158 if (!(rt->dst.dev->features&NETIF_F_SG))
1161 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1162 mtu = cork->fragsize;
1164 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
1165 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
1167 if (cork->length + size > 0xFFFF - fragheaderlen) {
1168 ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport, mtu);
1172 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
1175 cork->length += size;
1176 if ((size + skb->len > mtu) &&
1177 (sk->sk_protocol == IPPROTO_UDP) &&
1178 (rt->dst.dev->features & NETIF_F_UFO)) {
1179 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
1180 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1187 if (skb_is_gso(skb))
1191 /* Check if the remaining data fits into current packet. */
1192 len = mtu - skb->len;
1194 len = maxfraglen - skb->len;
1197 struct sk_buff *skb_prev;
1201 fraggap = skb_prev->len - maxfraglen;
1203 alloclen = fragheaderlen + hh_len + fraggap + 15;
1204 skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
1205 if (unlikely(!skb)) {
1211 * Fill in the control structures
1213 skb->ip_summed = CHECKSUM_NONE;
1215 skb_reserve(skb, hh_len);
1218 * Find where to start putting bytes.
1220 skb_put(skb, fragheaderlen + fraggap);
1221 skb_reset_network_header(skb);
1222 skb->transport_header = (skb->network_header +
1225 skb->csum = skb_copy_and_csum_bits(skb_prev,
1227 skb_transport_header(skb),
1229 skb_prev->csum = csum_sub(skb_prev->csum,
1231 pskb_trim_unique(skb_prev, maxfraglen);
1235 * Put the packet on the pending queue.
1237 __skb_queue_tail(&sk->sk_write_queue, skb);
1241 i = skb_shinfo(skb)->nr_frags;
1244 if (skb_can_coalesce(skb, i, page, offset)) {
1245 skb_frag_size_add(&skb_shinfo(skb)->frags[i-1], len);
1246 } else if (i < MAX_SKB_FRAGS) {
1248 skb_fill_page_desc(skb, i, page, offset, len);
1254 if (skb->ip_summed == CHECKSUM_NONE) {
1256 csum = csum_page(page, offset, len);
1257 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1261 skb->data_len += len;
1262 skb->truesize += len;
1263 atomic_add(len, &sk->sk_wmem_alloc);
1270 cork->length -= size;
1271 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
1275 static void ip_cork_release(struct inet_cork *cork)
1277 cork->flags &= ~IPCORK_OPT;
1280 dst_release(cork->dst);
1285 * Combined all pending IP fragments on the socket as one IP datagram
1286 * and push them out.
1288 struct sk_buff *__ip_make_skb(struct sock *sk,
1290 struct sk_buff_head *queue,
1291 struct inet_cork *cork)
1293 struct sk_buff *skb, *tmp_skb;
1294 struct sk_buff **tail_skb;
1295 struct inet_sock *inet = inet_sk(sk);
1296 struct net *net = sock_net(sk);
1297 struct ip_options *opt = NULL;
1298 struct rtable *rt = (struct rtable *)cork->dst;
1303 if ((skb = __skb_dequeue(queue)) == NULL)
1305 tail_skb = &(skb_shinfo(skb)->frag_list);
1307 /* move skb->data to ip header from ext header */
1308 if (skb->data < skb_network_header(skb))
1309 __skb_pull(skb, skb_network_offset(skb));
1310 while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1311 __skb_pull(tmp_skb, skb_network_header_len(skb));
1312 *tail_skb = tmp_skb;
1313 tail_skb = &(tmp_skb->next);
1314 skb->len += tmp_skb->len;
1315 skb->data_len += tmp_skb->len;
1316 skb->truesize += tmp_skb->truesize;
1317 tmp_skb->destructor = NULL;
1321 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1322 * to fragment the frame generated here. No matter, what transforms
1323 * how transforms change size of the packet, it will come out.
1325 if (inet->pmtudisc < IP_PMTUDISC_DO)
1328 /* DF bit is set when we want to see DF on outgoing frames.
1329 * If local_df is set too, we still allow to fragment this frame
1331 if (inet->pmtudisc >= IP_PMTUDISC_DO ||
1332 (skb->len <= dst_mtu(&rt->dst) &&
1333 ip_dont_fragment(sk, &rt->dst)))
1336 if (cork->flags & IPCORK_OPT)
1339 if (rt->rt_type == RTN_MULTICAST)
1342 ttl = ip_select_ttl(inet, &rt->dst);
1347 iph->tos = inet->tos;
1349 ip_select_ident(skb, sk);
1351 iph->protocol = sk->sk_protocol;
1352 ip_copy_addrs(iph, fl4);
1355 iph->ihl += opt->optlen>>2;
1356 ip_options_build(skb, opt, cork->addr, rt, 0);
1359 skb->priority = sk->sk_priority;
1360 skb->mark = sk->sk_mark;
1362 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
1366 skb_dst_set(skb, &rt->dst);
1368 if (iph->protocol == IPPROTO_ICMP)
1369 icmp_out_count(net, ((struct icmphdr *)
1370 skb_transport_header(skb))->type);
1372 ip_cork_release(cork);
1377 int ip_send_skb(struct sk_buff *skb)
1379 struct net *net = sock_net(skb->sk);
1382 err = ip_local_out(skb);
1385 err = net_xmit_errno(err);
1387 IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
1393 int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4)
1395 struct sk_buff *skb;
1397 skb = ip_finish_skb(sk, fl4);
1401 /* Netfilter gets whole the not fragmented skb. */
1402 return ip_send_skb(skb);
1406 * Throw away all pending data on the socket.
1408 static void __ip_flush_pending_frames(struct sock *sk,
1409 struct sk_buff_head *queue,
1410 struct inet_cork *cork)
1412 struct sk_buff *skb;
1414 while ((skb = __skb_dequeue_tail(queue)) != NULL)
1417 ip_cork_release(cork);
1420 void ip_flush_pending_frames(struct sock *sk)
1422 __ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base);
1425 struct sk_buff *ip_make_skb(struct sock *sk,
1427 int getfrag(void *from, char *to, int offset,
1428 int len, int odd, struct sk_buff *skb),
1429 void *from, int length, int transhdrlen,
1430 struct ipcm_cookie *ipc, struct rtable **rtp,
1433 struct inet_cork cork;
1434 struct sk_buff_head queue;
1437 if (flags & MSG_PROBE)
1440 __skb_queue_head_init(&queue);
1445 err = ip_setup_cork(sk, &cork, ipc, rtp);
1447 return ERR_PTR(err);
1449 err = __ip_append_data(sk, fl4, &queue, &cork, getfrag,
1450 from, length, transhdrlen, flags);
1452 __ip_flush_pending_frames(sk, &queue, &cork);
1453 return ERR_PTR(err);
1456 return __ip_make_skb(sk, fl4, &queue, &cork);
1460 * Fetch data from kernel space and fill in checksum if needed.
1462 static int ip_reply_glue_bits(void *dptr, char *to, int offset,
1463 int len, int odd, struct sk_buff *skb)
1467 csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
1468 skb->csum = csum_block_add(skb->csum, csum, odd);
1473 * Generic function to send a packet as reply to another packet.
1474 * Used to send TCP resets so far. ICMP should use this function too.
1476 * Should run single threaded per socket because it uses the sock
1477 * structure to pass arguments.
1479 void ip_send_reply(struct sock *sk, struct sk_buff *skb, __be32 daddr,
1480 const struct ip_reply_arg *arg, unsigned int len)
1482 struct inet_sock *inet = inet_sk(sk);
1483 struct ip_options_data replyopts;
1484 struct ipcm_cookie ipc;
1486 struct rtable *rt = skb_rtable(skb);
1488 if (ip_options_echo(&replyopts.opt.opt, skb))
1495 if (replyopts.opt.opt.optlen) {
1496 ipc.opt = &replyopts.opt;
1498 if (replyopts.opt.opt.srr)
1499 daddr = replyopts.opt.opt.faddr;
1502 flowi4_init_output(&fl4, arg->bound_dev_if, 0,
1504 RT_SCOPE_UNIVERSE, sk->sk_protocol,
1505 ip_reply_arg_flowi_flags(arg),
1506 daddr, rt->rt_spec_dst,
1507 tcp_hdr(skb)->source, tcp_hdr(skb)->dest);
1508 security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));
1509 rt = ip_route_output_key(sock_net(sk), &fl4);
1513 /* And let IP do all the hard work.
1515 This chunk is not reenterable, hence spinlock.
1516 Note that it uses the fact, that this function is called
1517 with locally disabled BH and that sk cannot be already spinlocked.
1520 inet->tos = arg->tos;
1521 sk->sk_priority = skb->priority;
1522 sk->sk_protocol = ip_hdr(skb)->protocol;
1523 sk->sk_bound_dev_if = arg->bound_dev_if;
1524 ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
1525 &ipc, &rt, MSG_DONTWAIT);
1526 if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
1527 if (arg->csumoffset >= 0)
1528 *((__sum16 *)skb_transport_header(skb) +
1529 arg->csumoffset) = csum_fold(csum_add(skb->csum,
1531 skb->ip_summed = CHECKSUM_NONE;
1532 ip_push_pending_frames(sk, &fl4);
1540 void __init ip_init(void)
1545 #if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1546 igmp_mc_proc_init();