32 #define indirect_call_gro_receive_l4(f2, f1, cb, head, skb)	\  argument
34 	unlikely(gro_recursion_inc_test(skb)) ?			\
35 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
36 		INDIRECT_CALL_L4(cb, f2, f1, head, skb);	\
39 static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)  in ipv6_gso_pull_exthdrs()  argument
57 		if (unlikely(!pskb_may_pull(skb, 8)))  in ipv6_gso_pull_exthdrs()
60 		opth = (void *)skb->data;  in ipv6_gso_pull_exthdrs()
63 		if (unlikely(!pskb_may_pull(skb, len)))  in ipv6_gso_pull_exthdrs()
66 		opth = (void *)skb->data;  in ipv6_gso_pull_exthdrs()
68 		__skb_pull(skb, len);  in ipv6_gso_pull_exthdrs()
74 static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,  in ipv6_gso_segment()  argument
89 	skb_reset_network_header(skb);  in ipv6_gso_segment()
90 	nhoff = skb_network_header(skb) - skb_mac_header(skb);  in ipv6_gso_segment()
91 	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))  in ipv6_gso_segment()
94 	encap = SKB_GSO_CB(skb)->encap_level > 0;  in ipv6_gso_segment()
96 		features &= skb->dev->hw_enc_features;  in ipv6_gso_segment()
97 	SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);  in ipv6_gso_segment()
99 	ipv6h = ipv6_hdr(skb);  in ipv6_gso_segment()
100 	__skb_pull(skb, sizeof(*ipv6h));  in ipv6_gso_segment()
103 	proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);  in ipv6_gso_segment()
105 	if (skb->encapsulation &&  in ipv6_gso_segment()
106 	    skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))  in ipv6_gso_segment()
108 			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);  in ipv6_gso_segment()
110 		udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&  in ipv6_gso_segment()
111 			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);  in ipv6_gso_segment()
115 		skb_reset_transport_header(skb);  in ipv6_gso_segment()
116 		segs = ops->callbacks.gso_segment(skb, features);  in ipv6_gso_segment()
118 			skb->network_header = skb_mac_header(skb) + nhoff - skb->head;  in ipv6_gso_segment()
126 	for (skb = segs; skb; skb = skb->next) {  in ipv6_gso_segment()
127 		ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);  in ipv6_gso_segment()
128 		if (gso_partial && skb_is_gso(skb))  in ipv6_gso_segment()
129 			payload_len = skb_shinfo(skb)->gso_size +  in ipv6_gso_segment()
130 				      SKB_GSO_CB(skb)->data_offset +  in ipv6_gso_segment()
131 				      skb->head - (unsigned char *)(ipv6h + 1);  in ipv6_gso_segment()
133 			payload_len = skb->len - nhoff - sizeof(*ipv6h);  in ipv6_gso_segment()
135 		skb->network_header = (u8 *)ipv6h - skb->head;  in ipv6_gso_segment()
136 		skb_reset_mac_len(skb);  in ipv6_gso_segment()
139 			int err = ip6_find_1stfragopt(skb, &prevhdr);  in ipv6_gso_segment()
146 			if (skb->next)  in ipv6_gso_segment()
152 			skb_reset_inner_headers(skb);  in ipv6_gso_segment()
186 							 struct sk_buff *skb)  in ipv6_gro_receive()  argument
198 	off = skb_gro_offset(skb);  in ipv6_gro_receive()
200 	iph = skb_gro_header_fast(skb, off);  in ipv6_gro_receive()
201 	if (skb_gro_header_hard(skb, hlen)) {  in ipv6_gro_receive()
202 		iph = skb_gro_header_slow(skb, hlen, off);  in ipv6_gro_receive()
207 	skb_set_network_header(skb, off);  in ipv6_gro_receive()
208 	skb_gro_pull(skb, sizeof(*iph));  in ipv6_gro_receive()
209 	skb_set_transport_header(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
211 	flush += ntohs(iph->payload_len) != skb_gro_len(skb);  in ipv6_gro_receive()
217 		__pskb_pull(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
218 		skb_gro_frag0_invalidate(skb);  in ipv6_gro_receive()
219 		proto = ipv6_gso_pull_exthdrs(skb, proto);  in ipv6_gro_receive()
220 		skb_gro_pull(skb, -skb_transport_offset(skb));  in ipv6_gro_receive()
221 		skb_reset_transport_header(skb);  in ipv6_gro_receive()
222 		__skb_push(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
228 		iph = ipv6_hdr(skb);  in ipv6_gro_receive()
231 	NAPI_GRO_CB(skb)->proto = proto;  in ipv6_gro_receive()
234 	nlen = skb_network_header_len(skb);  in ipv6_gro_receive()
272 		if (NAPI_GRO_CB(skb)->is_atomic)  in ipv6_gro_receive()
276 	NAPI_GRO_CB(skb)->is_atomic = true;  in ipv6_gro_receive()
277 	NAPI_GRO_CB(skb)->flush |= flush;  in ipv6_gro_receive()
279 	skb_gro_postpull_rcsum(skb, iph, nlen);  in ipv6_gro_receive()
282 					 ops->callbacks.gro_receive, head, skb);  in ipv6_gro_receive()
288 	skb_gro_flush_final(skb, pp, flush);  in ipv6_gro_receive()
294 					      struct sk_buff *skb)  in sit_ip6ip6_gro_receive()  argument
298 	if (NAPI_GRO_CB(skb)->encap_mark) {  in sit_ip6ip6_gro_receive()
299 		NAPI_GRO_CB(skb)->flush = 1;  in sit_ip6ip6_gro_receive()
303 	NAPI_GRO_CB(skb)->encap_mark = 1;  in sit_ip6ip6_gro_receive()
305 	return ipv6_gro_receive(head, skb);  in sit_ip6ip6_gro_receive()
309 					  struct sk_buff *skb)  in ip4ip6_gro_receive()  argument
313 	if (NAPI_GRO_CB(skb)->encap_mark) {  in ip4ip6_gro_receive()
314 		NAPI_GRO_CB(skb)->flush = 1;  in ip4ip6_gro_receive()
318 	NAPI_GRO_CB(skb)->encap_mark = 1;  in ip4ip6_gro_receive()
320 	return inet_gro_receive(head, skb);  in ip4ip6_gro_receive()
323 INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)  in ipv6_gro_complete()  argument
326 	struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + nhoff);  in ipv6_gro_complete()
329 	if (skb->encapsulation) {  in ipv6_gro_complete()
330 		skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));  in ipv6_gro_complete()
331 		skb_set_inner_network_header(skb, nhoff);  in ipv6_gro_complete()
334 	iph->payload_len = htons(skb->len - nhoff - sizeof(*iph));  in ipv6_gro_complete()
343 			       udp6_gro_complete, skb, nhoff);  in ipv6_gro_complete()
351 static int sit_gro_complete(struct sk_buff *skb, int nhoff)  in sit_gro_complete()  argument
353 	skb->encapsulation = 1;  in sit_gro_complete()
354 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;  in sit_gro_complete()
355 	return ipv6_gro_complete(skb, nhoff);  in sit_gro_complete()
358 static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)  in ip6ip6_gro_complete()  argument
360 	skb->encapsulation = 1;  in ip6ip6_gro_complete()
361 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;  in ip6ip6_gro_complete()
362 	return ipv6_gro_complete(skb, nhoff);  in ip6ip6_gro_complete()
365 static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)  in ip4ip6_gro_complete()  argument
367 	skb->encapsulation = 1;  in ip4ip6_gro_complete()
368 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;  in ip4ip6_gro_complete()
369 	return inet_gro_complete(skb, nhoff);  in ip4ip6_gro_complete()
381 static struct sk_buff *sit_gso_segment(struct sk_buff *skb,  in sit_gso_segment()  argument
384 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))  in sit_gso_segment()
387 	return ipv6_gso_segment(skb, features);  in sit_gso_segment()
390 static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,  in ip4ip6_gso_segment()  argument
393 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))  in ip4ip6_gso_segment()
396 	return inet_gso_segment(skb, features);  in ip4ip6_gso_segment()
399 static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,  in ip6ip6_gso_segment()  argument
402 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))  in ip6ip6_gso_segment()
405 	return ipv6_gso_segment(skb, features);  in ip6ip6_gso_segment()