1346 static unsigned int fanout_demux_hash(struct packet_fanout *f,  in fanout_demux_hash()  argument
1353 static unsigned int fanout_demux_lb(struct packet_fanout *f,  in fanout_demux_lb()  argument
1357 	unsigned int val = atomic_inc_return(&f->rr_cur);  in fanout_demux_lb()
1362 static unsigned int fanout_demux_cpu(struct packet_fanout *f,  in fanout_demux_cpu()  argument
1369 static unsigned int fanout_demux_rnd(struct packet_fanout *f,  in fanout_demux_rnd()  argument
1376 static unsigned int fanout_demux_rollover(struct packet_fanout *f,  in fanout_demux_rollover()  argument
1384 	po = pkt_sk(f->arr[idx]);  in fanout_demux_rollover()
1396 		po_next = pkt_sk(f->arr[i]);  in fanout_demux_rollover()
1415 static unsigned int fanout_demux_qm(struct packet_fanout *f,  in fanout_demux_qm()  argument
1422 static unsigned int fanout_demux_bpf(struct packet_fanout *f,  in fanout_demux_bpf()  argument
1430 	prog = rcu_dereference(f->bpf_prog);  in fanout_demux_bpf()
1438 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)  in fanout_has_flag()  argument
1440 	return f->flags & (flag >> 8);  in fanout_has_flag()
1446 	struct packet_fanout *f = pt->af_packet_priv;  in packet_rcv_fanout()  local
1447 	unsigned int num = READ_ONCE(f->num_members);  in packet_rcv_fanout()
1448 	struct net *net = read_pnet(&f->net);  in packet_rcv_fanout()
1457 	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {  in packet_rcv_fanout()
1462 	switch (f->type) {  in packet_rcv_fanout()
1465 		idx = fanout_demux_hash(f, skb, num);  in packet_rcv_fanout()
1468 		idx = fanout_demux_lb(f, skb, num);  in packet_rcv_fanout()
1471 		idx = fanout_demux_cpu(f, skb, num);  in packet_rcv_fanout()
1474 		idx = fanout_demux_rnd(f, skb, num);  in packet_rcv_fanout()
1477 		idx = fanout_demux_qm(f, skb, num);  in packet_rcv_fanout()
1480 		idx = fanout_demux_rollover(f, skb, 0, false, num);  in packet_rcv_fanout()
1484 		idx = fanout_demux_bpf(f, skb, num);  in packet_rcv_fanout()
1488 	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))  in packet_rcv_fanout()
1489 		idx = fanout_demux_rollover(f, skb, idx, true, num);  in packet_rcv_fanout()
1491 	po = pkt_sk(f->arr[idx]);  in packet_rcv_fanout()
1501 	struct packet_fanout *f = po->fanout;  in __fanout_link()  local
1503 	spin_lock(&f->lock);  in __fanout_link()
1504 	f->arr[f->num_members] = sk;  in __fanout_link()
1506 	f->num_members++;  in __fanout_link()
1507 	if (f->num_members == 1)  in __fanout_link()
1508 		dev_add_pack(&f->prot_hook);  in __fanout_link()
1509 	spin_unlock(&f->lock);  in __fanout_link()
1514 	struct packet_fanout *f = po->fanout;  in __fanout_unlink()  local
1517 	spin_lock(&f->lock);  in __fanout_unlink()
1518 	for (i = 0; i < f->num_members; i++) {  in __fanout_unlink()
1519 		if (f->arr[i] == sk)  in __fanout_unlink()
1522 	BUG_ON(i >= f->num_members);  in __fanout_unlink()
1523 	f->arr[i] = f->arr[f->num_members - 1];  in __fanout_unlink()
1524 	f->num_members--;  in __fanout_unlink()
1525 	if (f->num_members == 0)  in __fanout_unlink()
1526 		__dev_remove_pack(&f->prot_hook);  in __fanout_unlink()
1527 	spin_unlock(&f->lock);  in __fanout_unlink()
1538 static void fanout_init_data(struct packet_fanout *f)  in fanout_init_data()  argument
1540 	switch (f->type) {  in fanout_init_data()
1542 		atomic_set(&f->rr_cur, 0);  in fanout_init_data()
1546 		RCU_INIT_POINTER(f->bpf_prog, NULL);  in fanout_init_data()
1551 static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)  in __fanout_set_data_bpf()  argument
1555 	spin_lock(&f->lock);  in __fanout_set_data_bpf()
1556 	old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));  in __fanout_set_data_bpf()
1557 	rcu_assign_pointer(f->bpf_prog, new);  in __fanout_set_data_bpf()
1558 	spin_unlock(&f->lock);  in __fanout_set_data_bpf()
1622 static void fanout_release_data(struct packet_fanout *f)  in fanout_release_data()  argument
1624 	switch (f->type) {  in fanout_release_data()
1627 		__fanout_set_data_bpf(f, NULL);  in fanout_release_data()
1635 	struct packet_fanout *f, *match;  in fanout_add()  local
1674 	list_for_each_entry(f, &fanout_list, list) {  in fanout_add()
1675 		if (f->id == id &&  in fanout_add()
1676 		    read_pnet(&f->net) == sock_net(sk)) {  in fanout_add()
1677 			match = f;  in fanout_add()
1743 	struct packet_fanout *f;  in fanout_release()  local
1746 	f = po->fanout;  in fanout_release()
1747 	if (f) {  in fanout_release()
1750 		if (atomic_dec_and_test(&f->sk_ref))  in fanout_release()
1751 			list_del(&f->list);  in fanout_release()
1753 			f = NULL;  in fanout_release()
1757 	return f;  in fanout_release()
2997 	struct packet_fanout *f;  in packet_release()  local
3039 	f = fanout_release(sk);  in packet_release()
3043 	if (f) {  in packet_release()
3045 		fanout_release_data(f);  in packet_release()
3046 		kfree(f);  in packet_release()