1 /* SPDX-License-Identifier: GPL-2.0 */
8  * RCU-protected list version
15  * purpose.  The list_empty() function fetches the RCU-protected pointer
23  * INIT_LIST_HEAD_RCU - Initialize a list_head visible to RCU readers
33 	WRITE_ONCE(list->next, list);  in INIT_LIST_HEAD_RCU()
34 	WRITE_ONCE(list->prev, list);  in INIT_LIST_HEAD_RCU()
38  * return the ->next pointer of a list_head in an rcu safe
41 #define list_next_rcu(list)	(*((struct list_head __rcu **)(&(list)->next)))
44  * list_tail_rcu - returns the prev pointer of the head of the list
51 #define list_tail_rcu(head)	(*((struct list_head __rcu **)(&(head)->prev)))
64 			 "RCU-list traversed in non-reader section!");	\
70 		"RCU-list traversed without holding the required lock!");\
91 	new->next = next;  in __list_add_rcu()
92 	new->prev = prev;  in __list_add_rcu()
94 	next->prev = new;  in __list_add_rcu()
98  * list_add_rcu - add a new entry to rcu-protected list
107  * with another list-mutation primitive, such as list_add_rcu()
110  * the _rcu list-traversal primitives, such as
115 	__list_add_rcu(new, head, head->next);  in list_add_rcu()
119  * list_add_tail_rcu - add a new entry to rcu-protected list
128  * with another list-mutation primitive, such as list_add_tail_rcu()
131  * the _rcu list-traversal primitives, such as
137 	__list_add_rcu(new, head->prev, head);  in list_add_tail_rcu()
141  * list_del_rcu - deletes entry from list without re-initialization
153  * with another list-mutation primitive, such as list_del_rcu()
156  * the _rcu list-traversal primitives, such as
167 	entry->prev = LIST_POISON2;  in list_del_rcu()
171  * hlist_del_init_rcu - deletes entry from hash list with re-initialization
185  * list-mutation primitive, such as hlist_add_head_rcu() or
187  * perfectly legal to run concurrently with the _rcu list-traversal
194 		WRITE_ONCE(n->pprev, NULL);  in hlist_del_init_rcu()
199  * list_replace_rcu - replace old entry by new one
209 	new->next = old->next;  in list_replace_rcu()
210 	new->prev = old->prev;  in list_replace_rcu()
211 	rcu_assign_pointer(list_next_rcu(new->prev), new);  in list_replace_rcu()
212 	new->next->prev = new;  in list_replace_rcu()
213 	old->prev = LIST_POISON2;  in list_replace_rcu()
217  * __list_splice_init_rcu - join an RCU-protected list into an existing list.
218  * @list:	the RCU-protected list to splice
221  * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
223  * The list pointed to by @prev and @next can be RCU-read traversed
230  * modify the list as soon as sync() begins execution. If this sort of thing
232  * created.  But only if -really- needed -- there is no shortage of RCU API
238 					  void (*sync)(void))  in __list_splice_init_rcu()
240 	struct list_head *first = list->next;  in __list_splice_init_rcu()
241 	struct list_head *last = list->prev;  in __list_splice_init_rcu()
258 	sync();  in __list_splice_init_rcu()
270 	last->next = next;  in __list_splice_init_rcu()
272 	first->prev = prev;  in __list_splice_init_rcu()
273 	next->prev = last;  in __list_splice_init_rcu()
277  * list_splice_init_rcu - splice an RCU-protected list into an existing list,
279  * @list:	the RCU-protected list to splice
281  * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
285 					void (*sync)(void))  in list_splice_init_rcu()
288 		__list_splice_init_rcu(list, head, head->next, sync);  in list_splice_init_rcu()
292  * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
294  * @list:	the RCU-protected list to splice
296  * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
300 					     void (*sync)(void))  in list_splice_tail_init_rcu()
303 		__list_splice_init_rcu(list, head->prev, head, sync);  in list_splice_tail_init_rcu()
307  * list_entry_rcu - get the struct for this entry
309  * @type:       the type of the struct this is embedded in.
312  * This primitive may safely run concurrently with the _rcu list-mutation
331  * list_first_entry_rcu rereads the ->next pointer.
333  * Rereading the ->next pointer is not a problem for list_empty() and
341  * list_first_or_null_rcu - get the first element from a list
343  * @type:       the type of the struct this is embedded in.
348  * This primitive may safely run concurrently with the _rcu list-mutation
354 	struct list_head *__next = READ_ONCE(__ptr->next); \
359  * list_next_or_null_rcu - get the first element from a list
362  * @type:       the type of the struct this is embedded in.
367  * This primitive may safely run concurrently with the _rcu list-mutation
374 	struct list_head *__next = READ_ONCE(__ptr->next); \
380  * list_for_each_entry_rcu	-	iterate over rcu list of given type
384  * @cond:	optional lockdep expression if called from non-RCU protection.
386  * This list-traversal primitive may safely run concurrently with
387  * the _rcu list-mutation primitives such as list_add_rcu()
392 	     pos = list_entry_rcu((head)->next, typeof(*pos), member);	\
393 		&pos->member != (head);					\
394 		pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
397  * list_for_each_entry_srcu	-	iterate over rcu list of given type
403  * This list-traversal primitive may safely run concurrently with
404  * the _rcu list-mutation primitives such as list_add_rcu()
411 	     pos = list_entry_rcu((head)->next, typeof(*pos), member);	\
412 		&pos->member != (head);					\
413 		pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
416  * list_entry_lockless - get the struct for this entry
418  * @type:       the type of the struct this is embedded in.
422  * list-mutation primitives such as list_add_rcu(), but requires some
423  * implicit RCU read-side guarding.  One example is running within a special
424  * exception-time environment where preemption is disabled and where lockdep
432  * list_for_each_entry_lockless - iterate over rcu list of given type
438  * list-mutation primitives such as list_add_rcu(), but requires some
439  * implicit RCU read-side guarding.  One example is running within a special
440  * exception-time environment where preemption is disabled and where lockdep
445 	for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
446 	     &pos->member != (head); \
447 	     pos = list_entry_lockless(pos->member.next, typeof(*pos), member))
450  * list_for_each_entry_continue_rcu - continue iteration over list of given type
459  * previous walk of the list in the same RCU read-side critical section, or
460  * that you held some sort of non-RCU reference (such as a reference count)
468 	for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
469 	     &pos->member != (head);	\
470 	     pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
473  * list_for_each_entry_from_rcu - iterate over a list from current point
481  * previous walk of the list in the same RCU read-side critical section, or
482  * that you held some sort of non-RCU reference (such as a reference count)
490 	for (; &(pos)->member != (head);					\
491 		pos = list_entry_rcu(pos->member.next, typeof(*(pos)), member))
494  * hlist_del_rcu - deletes entry from hash list without re-initialization
506  * with another list-mutation primitive, such as hlist_add_head_rcu()
509  * the _rcu list-traversal primitives, such as
515 	WRITE_ONCE(n->pprev, LIST_POISON2);  in hlist_del_rcu()
519  * hlist_replace_rcu - replace old entry by new one
528 	struct hlist_node *next = old->next;  in hlist_replace_rcu()
530 	new->next = next;  in hlist_replace_rcu()
531 	WRITE_ONCE(new->pprev, old->pprev);  in hlist_replace_rcu()
532 	rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);  in hlist_replace_rcu()
534 		WRITE_ONCE(new->next->pprev, &new->next);  in hlist_replace_rcu()
535 	WRITE_ONCE(old->pprev, LIST_POISON2);  in hlist_replace_rcu()
539  * hlists_swap_heads_rcu - swap the lists the hlist heads point to
550 	struct hlist_node *node1 = left->first;  in hlists_swap_heads_rcu()
551 	struct hlist_node *node2 = right->first;  in hlists_swap_heads_rcu()
553 	rcu_assign_pointer(left->first, node2);  in hlists_swap_heads_rcu()
554 	rcu_assign_pointer(right->first, node1);  in hlists_swap_heads_rcu()
555 	WRITE_ONCE(node2->pprev, &left->first);  in hlists_swap_heads_rcu()
556 	WRITE_ONCE(node1->pprev, &right->first);  in hlists_swap_heads_rcu()
562 #define hlist_first_rcu(head)	(*((struct hlist_node __rcu **)(&(head)->first)))
563 #define hlist_next_rcu(node)	(*((struct hlist_node __rcu **)(&(node)->next)))
564 #define hlist_pprev_rcu(node)	(*((struct hlist_node __rcu **)((node)->pprev)))
577  * with another list-mutation primitive, such as hlist_add_head_rcu()
580  * the _rcu list-traversal primitives, such as
581  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
583  * list-traversal primitive must be guarded by rcu_read_lock().
588 	struct hlist_node *first = h->first;  in hlist_add_head_rcu()
590 	n->next = first;  in hlist_add_head_rcu()
591 	WRITE_ONCE(n->pprev, &h->first);  in hlist_add_head_rcu()
594 		WRITE_ONCE(first->pprev, &n->next);  in hlist_add_head_rcu()
608  * with another list-mutation primitive, such as hlist_add_head_rcu()
611  * the _rcu list-traversal primitives, such as
612  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
614  * list-traversal primitive must be guarded by rcu_read_lock().
622 	for (i = h->first; i; i = i->next)  in hlist_add_tail_rcu()
626 		n->next = last->next;  in hlist_add_tail_rcu()
627 		WRITE_ONCE(n->pprev, &last->next);  in hlist_add_tail_rcu()
645  * with another list-mutation primitive, such as hlist_add_head_rcu()
648  * the _rcu list-traversal primitives, such as
649  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
655 	WRITE_ONCE(n->pprev, next->pprev);  in hlist_add_before_rcu()
656 	n->next = next;  in hlist_add_before_rcu()
658 	WRITE_ONCE(next->pprev, &n->next);  in hlist_add_before_rcu()
672  * with another list-mutation primitive, such as hlist_add_head_rcu()
675  * the _rcu list-traversal primitives, such as
676  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
682 	n->next = prev->next;  in hlist_add_behind_rcu()
683 	WRITE_ONCE(n->pprev, &prev->next);  in hlist_add_behind_rcu()
685 	if (n->next)  in hlist_add_behind_rcu()
686 		WRITE_ONCE(n->next->pprev, &n->next);  in hlist_add_behind_rcu()
695  * hlist_for_each_entry_rcu - iterate over rcu list of given type
699  * @cond:	optional lockdep expression if called from non-RCU protection.
701  * This list-traversal primitive may safely run concurrently with
702  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
711 			&(pos)->member)), typeof(*(pos)), member))
714  * hlist_for_each_entry_srcu - iterate over rcu list of given type
720  * This list-traversal primitive may safely run concurrently with
721  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
732 			&(pos)->member)), typeof(*(pos)), member))
735  * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
740  * This list-traversal primitive may safely run concurrently with
741  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
752 			&(pos)->member)), typeof(*(pos)), member))
755  * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
760  * This list-traversal primitive may safely run concurrently with
761  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
769 			&(pos)->member)), typeof(*(pos)), member))
772  * hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point
778 			&(pos)->member)), typeof(*(pos)), member);	\
781 			&(pos)->member)), typeof(*(pos)), member))
784  * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
790 			&(pos)->member)), typeof(*(pos)), member);	\
793 			&(pos)->member)), typeof(*(pos)), member))
796  * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
803 			&(pos)->member)), typeof(*(pos)), member))