38 #define CSD_TYPE(_csd)	((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)
207 		return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */  in csd_lock_wait_getcpu()
236 	unsigned int flags = READ_ONCE(csd->node.u_flags);  in csd_lock_wait_toolong()
340 	smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));  in csd_lock_wait()
349 	smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));  in csd_lock_wait()
356 	csd->node.u_flags |= CSD_FLAG_LOCK;  in csd_lock()
368 	WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));  in csd_unlock()
373 	smp_store_release(&csd->node.u_flags, 0);  in csd_unlock()
378 void __smp_call_single_queue(int cpu, struct llist_node *node)  in __smp_call_single_queue()  argument
391 		csd = container_of(node, call_single_data_t, node.llist);  in __smp_call_single_queue()
410 	if (llist_add(node, &per_cpu(call_single_queue, cpu)))  in __smp_call_single_queue()
444 	__smp_call_single_queue(cpu, &csd->node.llist);  in generic_exec_single()
502 		llist_for_each_entry(csd, entry, node.llist) {  in __flush_smp_call_function_queue()
527 	llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {  in __flush_smp_call_function_queue()
534 				prev->next = &csd_next->node.llist;  in __flush_smp_call_function_queue()
536 				entry = &csd_next->node.llist;  in __flush_smp_call_function_queue()
544 			prev = &csd->node.llist;  in __flush_smp_call_function_queue()
555 	llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {  in __flush_smp_call_function_queue()
560 				prev->next = &csd_next->node.llist;  in __flush_smp_call_function_queue()
562 				entry = &csd_next->node.llist;  in __flush_smp_call_function_queue()
578 			prev = &csd->node.llist;  in __flush_smp_call_function_queue()
586 		csd = llist_entry(entry, typeof(*csd), node.llist);  in __flush_smp_call_function_queue()
635 		.node = { .u_flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, },  in smp_call_function_single()
672 	csd->node.src = smp_processor_id();  in smp_call_function_single()
673 	csd->node.dst = cpu;  in smp_call_function_single()
716 	if (csd->node.u_flags & CSD_FLAG_LOCK) {  in smp_call_function_single_async()
721 	csd->node.u_flags = CSD_FLAG_LOCK;  in smp_call_function_single_async()
844 				csd->node.u_flags |= CSD_TYPE_SYNC;  in smp_call_function_many_cond()
848 			csd->node.src = smp_processor_id();  in smp_call_function_many_cond()
849 			csd->node.dst = cpu;  in smp_call_function_many_cond()
853 			if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) {  in smp_call_function_many_cond()