1265 static unsigned int fanout_demux_hash(struct packet_fanout *f,  in fanout_demux_hash()  argument
1272 static unsigned int fanout_demux_lb(struct packet_fanout *f,  in fanout_demux_lb()  argument
1276 	unsigned int val = atomic_inc_return(&f->rr_cur);  in fanout_demux_lb()
1281 static unsigned int fanout_demux_cpu(struct packet_fanout *f,  in fanout_demux_cpu()  argument
1288 static unsigned int fanout_demux_rnd(struct packet_fanout *f,  in fanout_demux_rnd()  argument
1295 static unsigned int fanout_demux_rollover(struct packet_fanout *f,  in fanout_demux_rollover()  argument
1302 	i = j = min_t(int, f->next[idx], num - 1);  in fanout_demux_rollover()
1304 		if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {  in fanout_demux_rollover()
1306 				f->next[idx] = i;  in fanout_demux_rollover()
1316 static unsigned int fanout_demux_qm(struct packet_fanout *f,  in fanout_demux_qm()  argument
1323 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)  in fanout_has_flag()  argument
1325 	return f->flags & (flag >> 8);  in fanout_has_flag()
1331 	struct packet_fanout *f = pt->af_packet_priv;  in packet_rcv_fanout()  local
1332 	unsigned int num = READ_ONCE(f->num_members);  in packet_rcv_fanout()
1336 	if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||  in packet_rcv_fanout()
1342 	switch (f->type) {  in packet_rcv_fanout()
1345 		if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {  in packet_rcv_fanout()
1350 		idx = fanout_demux_hash(f, skb, num);  in packet_rcv_fanout()
1353 		idx = fanout_demux_lb(f, skb, num);  in packet_rcv_fanout()
1356 		idx = fanout_demux_cpu(f, skb, num);  in packet_rcv_fanout()
1359 		idx = fanout_demux_rnd(f, skb, num);  in packet_rcv_fanout()
1362 		idx = fanout_demux_qm(f, skb, num);  in packet_rcv_fanout()
1365 		idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);  in packet_rcv_fanout()
1369 	po = pkt_sk(f->arr[idx]);  in packet_rcv_fanout()
1370 	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&  in packet_rcv_fanout()
1372 		idx = fanout_demux_rollover(f, skb, idx, idx, num);  in packet_rcv_fanout()
1373 		po = pkt_sk(f->arr[idx]);  in packet_rcv_fanout()
1385 	struct packet_fanout *f = po->fanout;  in __fanout_link()  local
1387 	spin_lock(&f->lock);  in __fanout_link()
1388 	f->arr[f->num_members] = sk;  in __fanout_link()
1390 	f->num_members++;  in __fanout_link()
1391 	spin_unlock(&f->lock);  in __fanout_link()
1396 	struct packet_fanout *f = po->fanout;  in __fanout_unlink()  local
1399 	spin_lock(&f->lock);  in __fanout_unlink()
1400 	for (i = 0; i < f->num_members; i++) {  in __fanout_unlink()
1401 		if (f->arr[i] == sk)  in __fanout_unlink()
1404 	BUG_ON(i >= f->num_members);  in __fanout_unlink()
1405 	f->arr[i] = f->arr[f->num_members - 1];  in __fanout_unlink()
1406 	f->num_members--;  in __fanout_unlink()
1407 	spin_unlock(&f->lock);  in __fanout_unlink()
1421 	struct packet_fanout *f, *match;  in fanout_add()  local
1451 	list_for_each_entry(f, &fanout_list, list) {  in fanout_add()
1452 		if (f->id == id &&  in fanout_add()
1453 		    read_pnet(&f->net) == sock_net(sk)) {  in fanout_add()
1454 			match = f;  in fanout_add()
1513 	struct packet_fanout *f;  in fanout_release()  local
1516 	f = po->fanout;  in fanout_release()
1517 	if (f) {  in fanout_release()
1520 		if (atomic_dec_and_test(&f->sk_ref)) {  in fanout_release()
1521 			list_del(&f->list);  in fanout_release()
1522 			dev_remove_pack(&f->prot_hook);  in fanout_release()
1523 			kfree(f);  in fanout_release()