35 	rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
63 	return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);  in xfrm_dst_hash()
71 	return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);  in xfrm_src_hash()
78 	return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);  in xfrm_spi_hash()
119 	struct net *net = container_of(work, struct net, xfrm.state_hash_work);  in xfrm_hash_resize()
125 	nsize = xfrm_hash_new_size(net->xfrm.state_hmask);  in xfrm_hash_resize()
141 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
142 	write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
145 	odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);  in xfrm_hash_resize()
146 	for (i = net->xfrm.state_hmask; i >= 0; i--)  in xfrm_hash_resize()
149 	osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);  in xfrm_hash_resize()
150 	ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);  in xfrm_hash_resize()
151 	ohashmask = net->xfrm.state_hmask;  in xfrm_hash_resize()
153 	rcu_assign_pointer(net->xfrm.state_bydst, ndst);  in xfrm_hash_resize()
154 	rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);  in xfrm_hash_resize()
155 	rcu_assign_pointer(net->xfrm.state_byspi, nspi);  in xfrm_hash_resize()
156 	net->xfrm.state_hmask = nhashmask;  in xfrm_hash_resize()
158 	write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
159 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
663 		spin_lock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
669 		net->xfrm.state_num--;  in __xfrm_state_delete()
670 		spin_unlock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
707 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush_secctx_check()
710 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush_secctx_check()
727 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush_secctx_check()
731 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush_secctx_check()
762 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
768 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush()
771 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush()
775 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
787 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
793 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
805 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
811 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush()
815 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush()
820 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
829 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
838 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
845 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
846 	si->sadcnt = net->xfrm.state_num;  in xfrm_sad_getinfo()
847 	si->sadhcnt = net->xfrm.state_hmask + 1;  in xfrm_sad_getinfo()
849 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
937 	hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {  in __xfrm_state_lookup()
962 	hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) {  in __xfrm_state_lookup_byaddr()
998 	    (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&  in xfrm_hash_grow_check()
999 	    net->xfrm.state_num > net->xfrm.state_hmask)  in xfrm_hash_grow_check()
1000 		schedule_work(&net->xfrm.state_hash_work);  in xfrm_hash_grow_check()
1065 	sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_state_find()
1069 	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {  in xfrm_state_find()
1086 	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {  in xfrm_state_find()
1141 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1143 			list_add(&x->km.all, &net->xfrm.state_all);  in xfrm_state_find()
1144 			hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);  in xfrm_state_find()
1146 			hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);  in xfrm_state_find()
1149 				hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);  in xfrm_state_find()
1151 			x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in xfrm_state_find()
1153 				      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in xfrm_state_find()
1155 			net->xfrm.state_num++;  in xfrm_state_find()
1157 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1178 	if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {  in xfrm_state_find()
1197 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1199 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_stateonly_find()
1216 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1229 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1230 	list_for_each_entry(w, &net->xfrm.state_all, all) {  in xfrm_state_lookup_byspi()
1237 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1240 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1250 	list_add(&x->km.all, &net->xfrm.state_all);  in __xfrm_state_insert()
1254 	hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);  in __xfrm_state_insert()
1257 	hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);  in __xfrm_state_insert()
1263 		hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);  in __xfrm_state_insert()
1270 	net->xfrm.state_num++;  in __xfrm_state_insert()
1287 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __xfrm_state_bump_genids()
1302 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1305 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1322 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __find_acq_core()
1371 		x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in __find_acq_core()
1374 			      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in __find_acq_core()
1376 		list_add(&x->km.all, &net->xfrm.state_all);  in __find_acq_core()
1377 		hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);  in __find_acq_core()
1379 		hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);  in __find_acq_core()
1381 		net->xfrm.state_num++;  in __find_acq_core()
1404 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1433 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1575 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
1580 		hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_migrate_state_find()
1599 		hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {  in xfrm_migrate_state_find()
1615 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
1666 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1686 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1723 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1732 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1791 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
1793 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
1805 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
1807 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
1945 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in __xfrm_find_acq_byseq()
1948 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in __xfrm_find_acq_byseq()
1964 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
1966 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2049 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2052 		hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);  in xfrm_alloc_spi()
2053 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2093 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2095 		x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);  in xfrm_state_walk()
2098 	list_for_each_entry_from(x, &net->xfrm.state_all, all) {  in xfrm_state_walk()
2119 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2142 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2144 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2658 	INIT_LIST_HEAD(&net->xfrm.state_all);  in xfrm_state_init()
2662 	net->xfrm.state_bydst = xfrm_hash_alloc(sz);  in xfrm_state_init()
2663 	if (!net->xfrm.state_bydst)  in xfrm_state_init()
2665 	net->xfrm.state_bysrc = xfrm_hash_alloc(sz);  in xfrm_state_init()
2666 	if (!net->xfrm.state_bysrc)  in xfrm_state_init()
2668 	net->xfrm.state_byspi = xfrm_hash_alloc(sz);  in xfrm_state_init()
2669 	if (!net->xfrm.state_byspi)  in xfrm_state_init()
2671 	net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);  in xfrm_state_init()
2673 	net->xfrm.state_num = 0;  in xfrm_state_init()
2674 	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);  in xfrm_state_init()
2675 	spin_lock_init(&net->xfrm.xfrm_state_lock);  in xfrm_state_init()
2676 	seqcount_init(&net->xfrm.xfrm_state_hash_generation);  in xfrm_state_init()
2680 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_init()
2682 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_init()
2691 	flush_work(&net->xfrm.state_hash_work);  in xfrm_state_fini()
2695 	WARN_ON(!list_empty(&net->xfrm.state_all));  in xfrm_state_fini()
2697 	sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);  in xfrm_state_fini()
2698 	WARN_ON(!hlist_empty(net->xfrm.state_byspi));  in xfrm_state_fini()
2699 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_fini()
2700 	WARN_ON(!hlist_empty(net->xfrm.state_bysrc));  in xfrm_state_fini()
2701 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_fini()
2702 	WARN_ON(!hlist_empty(net->xfrm.state_bydst));  in xfrm_state_fini()
2703 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_fini()