44 	atomic_long_set(&lock->owner, 0);  in __mutex_init()
77 	return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);  in __mutex_owner()
80 static inline struct task_struct *__owner_task(unsigned long owner)  in __owner_task()  argument
82 	return (struct task_struct *)(owner & ~MUTEX_FLAGS);  in __owner_task()
101 static inline unsigned long __owner_flags(unsigned long owner)  in __owner_flags()  argument
103 	return owner & MUTEX_FLAGS;  in __owner_flags()
111 	unsigned long owner, curr = (unsigned long)current;  in __mutex_trylock_or_owner()  local
113 	owner = atomic_long_read(&lock->owner);  in __mutex_trylock_or_owner()
115 		unsigned long old, flags = __owner_flags(owner);  in __mutex_trylock_or_owner()
116 		unsigned long task = owner & ~MUTEX_FLAGS;  in __mutex_trylock_or_owner()
139 		old = atomic_long_cmpxchg_acquire(&lock->owner, owner, curr | flags);  in __mutex_trylock_or_owner()
140 		if (old == owner)  in __mutex_trylock_or_owner()
143 		owner = old;  in __mutex_trylock_or_owner()
146 	return __owner_task(owner);  in __mutex_trylock_or_owner()
173 	if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) {  in __mutex_trylock_fast()
185 	if (atomic_long_cmpxchg_release(&lock->owner, curr, 0UL) == curr)  in __mutex_unlock_fast()
194 	atomic_long_or(flag, &lock->owner);  in __mutex_set_flag()
199 	atomic_long_andnot(flag, &lock->owner);  in __mutex_clear_flag()
243 	unsigned long owner = atomic_long_read(&lock->owner);  in __mutex_handoff()  local
249 		DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);  in __mutex_handoff()
250 		DEBUG_LOCKS_WARN_ON(owner & MUTEX_FLAG_PICKUP);  in __mutex_handoff()
253 		new = (owner & MUTEX_FLAG_WAITERS);  in __mutex_handoff()
258 		old = atomic_long_cmpxchg_release(&lock->owner, owner, new);  in __mutex_handoff()
259 		if (old == owner)  in __mutex_handoff()
262 		owner = old;  in __mutex_handoff()
410 	struct task_struct *owner = __mutex_owner(lock);  in __ww_mutex_wound()  local
427 	if (!owner)  in __ww_mutex_wound()
439 		if (owner != current)  in __ww_mutex_wound()
440 			wake_up_process(owner);  in __ww_mutex_wound()
504 	if (likely(!(atomic_long_read(&lock->base.owner) & MUTEX_FLAG_WAITERS)))  in ww_mutex_set_context_fastpath()
547 	if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS))  in ww_mutex_spin_on_owner()
567 bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,  in mutex_spin_on_owner()  argument
575 	while (__mutex_owner(lock) == owner) {  in mutex_spin_on_owner()
592 		if (!owner->on_cpu || need_resched() ||  in mutex_spin_on_owner()
593 				vcpu_is_preempted(task_cpu(owner))) {  in mutex_spin_on_owner()
615 	struct task_struct *owner;  in mutex_can_spin_on_owner()  local
622 	owner = __mutex_owner(lock);  in mutex_can_spin_on_owner()
628 	if (owner)  in mutex_can_spin_on_owner()
629 		retval = owner->on_cpu && !vcpu_is_preempted(task_cpu(owner));  in mutex_can_spin_on_owner()
687 		struct task_struct *owner;  in mutex_optimistic_spin()  local
690 		owner = __mutex_trylock_or_owner(lock);  in mutex_optimistic_spin()
691 		if (!owner)  in mutex_optimistic_spin()
698 		if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))  in mutex_optimistic_spin()
1257 	unsigned long owner;  in __mutex_unlock_slowpath()  local
1268 	owner = atomic_long_read(&lock->owner);  in __mutex_unlock_slowpath()
1273 		DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);  in __mutex_unlock_slowpath()
1274 		DEBUG_LOCKS_WARN_ON(owner & MUTEX_FLAG_PICKUP);  in __mutex_unlock_slowpath()
1277 		if (owner & MUTEX_FLAG_HANDOFF)  in __mutex_unlock_slowpath()
1280 		old = atomic_long_cmpxchg_release(&lock->owner, owner,  in __mutex_unlock_slowpath()
1281 						  __owner_flags(owner));  in __mutex_unlock_slowpath()
1282 		if (old == owner) {  in __mutex_unlock_slowpath()
1283 			if (owner & MUTEX_FLAG_WAITERS)  in __mutex_unlock_slowpath()
1289 		owner = old;  in __mutex_unlock_slowpath()
1306 	if (owner & MUTEX_FLAG_HANDOFF)  in __mutex_unlock_slowpath()