28 static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb);
64 static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_transport_output()  argument
71 	skb_set_network_header(skb, -x->props.header_len);  in xfrm4_transport_output()
85 static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_transport_output()  argument
95 	hdr_len = x->type->hdr_offset(x, skb, &prevhdr);  in xfrm6_transport_output()
99 			   (prevhdr - x->props.header_len) - skb->data);  in xfrm6_transport_output()
100 	skb_set_network_header(skb, -x->props.header_len);  in xfrm6_transport_output()
116 static int xfrm6_ro_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_ro_output()  argument
125 	hdr_len = x->type->hdr_offset(x, skb, &prevhdr);  in xfrm6_ro_output()
129 			   (prevhdr - x->props.header_len) - skb->data);  in xfrm6_ro_output()
130 	skb_set_network_header(skb, -x->props.header_len);  in xfrm6_ro_output()
135 	x->lastused = ktime_get_real_seconds();  in xfrm6_ro_output()
148 static int xfrm4_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_beet_encap_add()  argument
159 	skb_set_network_header(skb, -x->props.header_len - hdrlen +  in xfrm4_beet_encap_add()
161 	if (x->sel.family != AF_INET6)  in xfrm4_beet_encap_add()
187 	top_iph->saddr = x->props.saddr.a4;  in xfrm4_beet_encap_add()
188 	top_iph->daddr = x->id.daddr.a4;  in xfrm4_beet_encap_add()
197 static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_tunnel_encap_add()  argument
206 	skb_set_network_header(skb, -x->props.header_len);  in xfrm4_tunnel_encap_add()
218 	if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)  in xfrm4_tunnel_encap_add()
225 	flags = x->props.flags;  in xfrm4_tunnel_encap_add()
234 	top_iph->saddr = x->props.saddr.a4;  in xfrm4_tunnel_encap_add()
235 	top_iph->daddr = x->id.daddr.a4;  in xfrm4_tunnel_encap_add()
242 static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_tunnel_encap_add()  argument
251 	skb_set_network_header(skb, -x->props.header_len);  in xfrm6_tunnel_encap_add()
263 	if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)  in xfrm6_tunnel_encap_add()
268 	if (x->props.flags & XFRM_STATE_NOECN)  in xfrm6_tunnel_encap_add()
272 	top_iph->saddr = *(struct in6_addr *)&x->props.saddr;  in xfrm6_tunnel_encap_add()
273 	top_iph->daddr = *(struct in6_addr *)&x->id.daddr;  in xfrm6_tunnel_encap_add()
277 static int xfrm6_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_beet_encap_add()  argument
288 	skb_set_network_header(skb, -x->props.header_len - hdr_len);  in xfrm6_beet_encap_add()
289 	if (x->sel.family != AF_INET6)  in xfrm6_beet_encap_add()
312 	top_iph->saddr = *(struct in6_addr *)&x->props.saddr;  in xfrm6_beet_encap_add()
313 	top_iph->daddr = *(struct in6_addr *)&x->id.daddr;  in xfrm6_beet_encap_add()
328 static int xfrm4_prepare_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_prepare_output()  argument
332 	err = xfrm_inner_extract_output(x, skb);  in xfrm4_prepare_output()
339 	switch (x->outer_mode.encap) {  in xfrm4_prepare_output()
341 		return xfrm4_beet_encap_add(x, skb);  in xfrm4_prepare_output()
343 		return xfrm4_tunnel_encap_add(x, skb);  in xfrm4_prepare_output()
350 static int xfrm6_prepare_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_prepare_output()  argument
355 	err = xfrm_inner_extract_output(x, skb);  in xfrm6_prepare_output()
362 	switch (x->outer_mode.encap) {  in xfrm6_prepare_output()
364 		return xfrm6_beet_encap_add(x, skb);  in xfrm6_prepare_output()
366 		return xfrm6_tunnel_encap_add(x, skb);  in xfrm6_prepare_output()
376 static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm_outer_mode_output()  argument
378 	switch (x->outer_mode.encap) {  in xfrm_outer_mode_output()
381 		if (x->outer_mode.family == AF_INET)  in xfrm_outer_mode_output()
382 			return xfrm4_prepare_output(x, skb);  in xfrm_outer_mode_output()
383 		if (x->outer_mode.family == AF_INET6)  in xfrm_outer_mode_output()
384 			return xfrm6_prepare_output(x, skb);  in xfrm_outer_mode_output()
387 		if (x->outer_mode.family == AF_INET)  in xfrm_outer_mode_output()
388 			return xfrm4_transport_output(x, skb);  in xfrm_outer_mode_output()
389 		if (x->outer_mode.family == AF_INET6)  in xfrm_outer_mode_output()
390 			return xfrm6_transport_output(x, skb);  in xfrm_outer_mode_output()
393 		if (x->outer_mode.family == AF_INET6)  in xfrm_outer_mode_output()
394 			return xfrm6_ro_output(x, skb);  in xfrm_outer_mode_output()
405 int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)  in pktgen_xfrm_outer_mode_output()  argument
407 	return xfrm_outer_mode_output(x, skb);  in pktgen_xfrm_outer_mode_output()
414 	struct xfrm_state *x = dst->xfrm;  in xfrm_output_one()  local
415 	struct net *net = xs_net(x);  in xfrm_output_one()
427 		skb->mark = xfrm_smark_get(skb->mark, x);  in xfrm_output_one()
429 		err = xfrm_outer_mode_output(x, skb);  in xfrm_output_one()
435 		spin_lock_bh(&x->lock);  in xfrm_output_one()
437 		if (unlikely(x->km.state != XFRM_STATE_VALID)) {  in xfrm_output_one()
443 		err = xfrm_state_check_expire(x);  in xfrm_output_one()
449 		err = x->repl->overflow(x, skb);  in xfrm_output_one()
455 		x->curlft.bytes += skb->len;  in xfrm_output_one()
456 		x->curlft.packets++;  in xfrm_output_one()
458 		spin_unlock_bh(&x->lock);  in xfrm_output_one()
468 			x->type_offload->encap(x, skb);  in xfrm_output_one()
473 			err = x->type->output(x, skb);  in xfrm_output_one()
491 		x = dst->xfrm;  in xfrm_output_one()
492 	} while (x && !(x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL));  in xfrm_output_one()
497 	spin_unlock_bh(&x->lock);  in xfrm_output_one()
569 	struct xfrm_state *x = skb_dst(skb)->xfrm;  in xfrm_output()  local
572 	switch (x->outer_mode.family) {  in xfrm_output()
586 	if (xfrm_dev_offload_ok(skb, x)) {  in xfrm_output()
598 		sp->xvec[sp->len++] = x;  in xfrm_output()
599 		xfrm_state_hold(x);  in xfrm_output()
609 		if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)  in xfrm_output()
657 static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_extract_output()  argument
661 	if (x->outer_mode.encap == XFRM_MODE_BEET &&  in xfrm4_extract_output()
709 static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_extract_output()  argument
728 static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm_inner_extract_output()  argument
732 	if (x->sel.family == AF_UNSPEC)  in xfrm_inner_extract_output()
733 		inner_mode = xfrm_ip2inner_mode(x,  in xfrm_inner_extract_output()
736 		inner_mode = &x->inner_mode;  in xfrm_inner_extract_output()
743 		return xfrm4_extract_output(x, skb);  in xfrm_inner_extract_output()
745 		return xfrm6_extract_output(x, skb);  in xfrm_inner_extract_output()