488 static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev *dst)  in dev_map_bpf_prog_run_skb()  argument
490 	struct xdp_txq_info txq = { .dev = dst->dev };  in dev_map_bpf_prog_run_skb()
494 	if (!dst->xdp_prog)  in dev_map_bpf_prog_run_skb()
500 	act = bpf_prog_run_generic_xdp(skb, &xdp, dst->xdp_prog);  in dev_map_bpf_prog_run_skb()
506 		bpf_warn_invalid_xdp_action(NULL, dst->xdp_prog, act);  in dev_map_bpf_prog_run_skb()
509 		trace_xdp_exception(dst->dev, dst->xdp_prog, act);  in dev_map_bpf_prog_run_skb()
525 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,  in dev_map_enqueue()  argument
528 	struct net_device *dev = dst->dev;  in dev_map_enqueue()
530 	return __xdp_enqueue(dev, xdpf, dev_rx, dst->xdp_prog);  in dev_map_enqueue()
589 	struct bpf_dtab_netdev *dst, *last_dst = NULL;  in dev_map_enqueue_multi()  local
603 			dst = rcu_dereference_check(dtab->netdev_map[i],  in dev_map_enqueue_multi()
605 			if (!is_valid_dst(dst, xdpf))  in dev_map_enqueue_multi()
608 			if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))  in dev_map_enqueue_multi()
613 				last_dst = dst;  in dev_map_enqueue_multi()
621 			last_dst = dst;  in dev_map_enqueue_multi()
626 			hlist_for_each_entry_rcu(dst, head, index_hlist,  in dev_map_enqueue_multi()
628 				if (!is_valid_dst(dst, xdpf))  in dev_map_enqueue_multi()
632 							dst->dev->ifindex))  in dev_map_enqueue_multi()
637 					last_dst = dst;  in dev_map_enqueue_multi()
645 				last_dst = dst;  in dev_map_enqueue_multi()
659 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,  in dev_map_generic_redirect()  argument
664 	err = xdp_ok_fwd_dev(dst->dev, skb->len);  in dev_map_generic_redirect()
672 	if (dev_map_bpf_prog_run_skb(skb, dst) != XDP_PASS)  in dev_map_generic_redirect()
675 	skb->dev = dst->dev;  in dev_map_generic_redirect()
681 static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst,  in dev_map_redirect_clone()  argument
692 	err = dev_map_generic_redirect(dst, nskb, xdp_prog);  in dev_map_redirect_clone()
706 	struct bpf_dtab_netdev *dst, *last_dst = NULL;  in dev_map_redirect_multi()  local
721 			dst = rcu_dereference_check(dtab->netdev_map[i],  in dev_map_redirect_multi()
723 			if (!dst)  in dev_map_redirect_multi()
726 			if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))  in dev_map_redirect_multi()
731 				last_dst = dst;  in dev_map_redirect_multi()
739 			last_dst = dst;  in dev_map_redirect_multi()
745 			hlist_for_each_entry_safe(dst, next, head, index_hlist) {  in dev_map_redirect_multi()
746 				if (!dst)  in dev_map_redirect_multi()
750 							dst->dev->ifindex))  in dev_map_redirect_multi()
755 					last_dst = dst;  in dev_map_redirect_multi()
763 				last_dst = dst;  in dev_map_redirect_multi()