54 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner,  in rt_mutex_set_owner()  argument
59 	if (rt_mutex_has_waiters(lock))  in rt_mutex_set_owner()
62 	lock->owner = (struct task_struct *)val;  in rt_mutex_set_owner()
65 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)  in clear_rt_mutex_waiters()  argument
67 	lock->owner = (struct task_struct *)  in clear_rt_mutex_waiters()
68 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);  in clear_rt_mutex_waiters()
71 static void fixup_rt_mutex_waiters(struct rt_mutex *lock)  in fixup_rt_mutex_waiters()  argument
73 	if (!rt_mutex_has_waiters(lock))  in fixup_rt_mutex_waiters()
74 		clear_rt_mutex_waiters(lock);  in fixup_rt_mutex_waiters()
83 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)  in mark_rt_mutex_waiters()  argument
85 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in mark_rt_mutex_waiters()
93 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)  in mark_rt_mutex_waiters()  argument
95 	lock->owner = (struct task_struct *)  in mark_rt_mutex_waiters()
96 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);  in mark_rt_mutex_waiters()
162 	struct rt_mutex *lock;  in rt_mutex_adjust_prio_chain()  local
233 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
234 	if (!spin_trylock(&lock->wait_lock)) {  in rt_mutex_adjust_prio_chain()
241 	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {  in rt_mutex_adjust_prio_chain()
242 		debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);  in rt_mutex_adjust_prio_chain()
243 		spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
248 	top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
251 	plist_del(&waiter->list_entry, &lock->wait_list);  in rt_mutex_adjust_prio_chain()
253 	plist_add(&waiter->list_entry, &lock->wait_list);  in rt_mutex_adjust_prio_chain()
260 	task = rt_mutex_owner(lock);  in rt_mutex_adjust_prio_chain()
264 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
274 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
282 	top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
283 	spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
303 static inline int try_to_steal_lock(struct rt_mutex *lock)  in try_to_steal_lock()  argument
305 	struct task_struct *pendowner = rt_mutex_owner(lock);  in try_to_steal_lock()
309 	if (!rt_mutex_owner_pending(lock))  in try_to_steal_lock()
326 	if (likely(!rt_mutex_has_waiters(lock))) {  in try_to_steal_lock()
332 	next = rt_mutex_top_waiter(lock);  in try_to_steal_lock()
369 static int try_to_take_rt_mutex(struct rt_mutex *lock)  in try_to_take_rt_mutex()  argument
390 	mark_rt_mutex_waiters(lock);  in try_to_take_rt_mutex()
392 	if (rt_mutex_owner(lock) && !try_to_steal_lock(lock))  in try_to_take_rt_mutex()
396 	debug_rt_mutex_lock(lock);  in try_to_take_rt_mutex()
398 	rt_mutex_set_owner(lock, current, 0);  in try_to_take_rt_mutex()
400 	rt_mutex_deadlock_account_lock(lock, current);  in try_to_take_rt_mutex()
412 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,  in task_blocks_on_rt_mutex()  argument
416 	struct task_struct *owner = rt_mutex_owner(lock);  in task_blocks_on_rt_mutex()
424 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
429 	if (rt_mutex_has_waiters(lock))  in task_blocks_on_rt_mutex()
430 		top_waiter = rt_mutex_top_waiter(lock);  in task_blocks_on_rt_mutex()
431 	plist_add(&waiter->list_entry, &lock->wait_list);  in task_blocks_on_rt_mutex()
437 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
460 	spin_unlock(&lock->wait_lock);  in task_blocks_on_rt_mutex()
462 	res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,  in task_blocks_on_rt_mutex()
465 	spin_lock(&lock->wait_lock);  in task_blocks_on_rt_mutex()
478 static void wakeup_next_waiter(struct rt_mutex *lock)  in wakeup_next_waiter()  argument
486 	waiter = rt_mutex_top_waiter(lock);  in wakeup_next_waiter()
487 	plist_del(&waiter->list_entry, &lock->wait_list);  in wakeup_next_waiter()
499 	rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING);  in wakeup_next_waiter()
514 	WARN_ON(pendowner->pi_blocked_on->lock != lock);  in wakeup_next_waiter()
518 	if (rt_mutex_has_waiters(lock)) {  in wakeup_next_waiter()
521 		next = rt_mutex_top_waiter(lock);  in wakeup_next_waiter()
534 static void remove_waiter(struct rt_mutex *lock,  in remove_waiter()  argument
537 	int first = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
538 	struct task_struct *owner = rt_mutex_owner(lock);  in remove_waiter()
543 	plist_del(&waiter->list_entry, &lock->wait_list);  in remove_waiter()
554 		if (rt_mutex_has_waiters(lock)) {  in remove_waiter()
557 			next = rt_mutex_top_waiter(lock);  in remove_waiter()
576 	spin_unlock(&lock->wait_lock);  in remove_waiter()
578 	rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);  in remove_waiter()
580 	spin_lock(&lock->wait_lock);  in remove_waiter()
612 rt_mutex_slowlock(struct rt_mutex *lock, int state,  in rt_mutex_slowlock()  argument
622 	spin_lock(&lock->wait_lock);  in rt_mutex_slowlock()
625 	if (try_to_take_rt_mutex(lock)) {  in rt_mutex_slowlock()
626 		spin_unlock(&lock->wait_lock);  in rt_mutex_slowlock()
641 		if (try_to_take_rt_mutex(lock))  in rt_mutex_slowlock()
664 			ret = task_blocks_on_rt_mutex(lock, &waiter,  in rt_mutex_slowlock()
685 		spin_unlock(&lock->wait_lock);  in rt_mutex_slowlock()
690 			schedule_rt_mutex(lock);  in rt_mutex_slowlock()
692 		spin_lock(&lock->wait_lock);  in rt_mutex_slowlock()
699 		remove_waiter(lock, &waiter);  in rt_mutex_slowlock()
705 	fixup_rt_mutex_waiters(lock);  in rt_mutex_slowlock()
707 	spin_unlock(&lock->wait_lock);  in rt_mutex_slowlock()
730 rt_mutex_slowtrylock(struct rt_mutex *lock)  in rt_mutex_slowtrylock()  argument
734 	spin_lock(&lock->wait_lock);  in rt_mutex_slowtrylock()
736 	if (likely(rt_mutex_owner(lock) != current)) {  in rt_mutex_slowtrylock()
738 		ret = try_to_take_rt_mutex(lock);  in rt_mutex_slowtrylock()
743 		fixup_rt_mutex_waiters(lock);  in rt_mutex_slowtrylock()
746 	spin_unlock(&lock->wait_lock);  in rt_mutex_slowtrylock()
755 rt_mutex_slowunlock(struct rt_mutex *lock)  in rt_mutex_slowunlock()  argument
757 	spin_lock(&lock->wait_lock);  in rt_mutex_slowunlock()
759 	debug_rt_mutex_unlock(lock);  in rt_mutex_slowunlock()
763 	if (!rt_mutex_has_waiters(lock)) {  in rt_mutex_slowunlock()
764 		lock->owner = NULL;  in rt_mutex_slowunlock()
765 		spin_unlock(&lock->wait_lock);  in rt_mutex_slowunlock()
769 	wakeup_next_waiter(lock);  in rt_mutex_slowunlock()
771 	spin_unlock(&lock->wait_lock);  in rt_mutex_slowunlock()
784 rt_mutex_fastlock(struct rt_mutex *lock, int state,  in rt_mutex_fastlock()  argument
786 		  int (*slowfn)(struct rt_mutex *lock, int state,  in rt_mutex_fastlock()  argument
790 	if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {  in rt_mutex_fastlock()
791 		rt_mutex_deadlock_account_lock(lock, current);  in rt_mutex_fastlock()
794 		return slowfn(lock, state, NULL, detect_deadlock);  in rt_mutex_fastlock()
798 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,  in rt_mutex_timed_fastlock()  argument
800 			int (*slowfn)(struct rt_mutex *lock, int state,  in rt_mutex_timed_fastlock()  argument
804 	if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {  in rt_mutex_timed_fastlock()
805 		rt_mutex_deadlock_account_lock(lock, current);  in rt_mutex_timed_fastlock()
808 		return slowfn(lock, state, timeout, detect_deadlock);  in rt_mutex_timed_fastlock()
812 rt_mutex_fasttrylock(struct rt_mutex *lock,  in rt_mutex_fasttrylock()  argument
813 		     int (*slowfn)(struct rt_mutex *lock))  in rt_mutex_fasttrylock()  argument
815 	if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {  in rt_mutex_fasttrylock()
816 		rt_mutex_deadlock_account_lock(lock, current);  in rt_mutex_fasttrylock()
819 	return slowfn(lock);  in rt_mutex_fasttrylock()
823 rt_mutex_fastunlock(struct rt_mutex *lock,  in rt_mutex_fastunlock()  argument
824 		    void (*slowfn)(struct rt_mutex *lock))  in rt_mutex_fastunlock()  argument
826 	if (likely(rt_mutex_cmpxchg(lock, current, NULL)))  in rt_mutex_fastunlock()
829 		slowfn(lock);  in rt_mutex_fastunlock()
837 void __sched rt_mutex_lock(struct rt_mutex *lock)  in rt_mutex_lock()  argument
841 	rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);  in rt_mutex_lock()
856 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,  in rt_mutex_lock_interruptible()  argument
861 	return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,  in rt_mutex_lock_interruptible()
882 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,  in rt_mutex_timed_lock()  argument
887 	return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,  in rt_mutex_timed_lock()
899 int __sched rt_mutex_trylock(struct rt_mutex *lock)  in rt_mutex_trylock()  argument
901 	return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);  in rt_mutex_trylock()
910 void __sched rt_mutex_unlock(struct rt_mutex *lock)  in rt_mutex_unlock()  argument
912 	rt_mutex_fastunlock(lock, rt_mutex_slowunlock);  in rt_mutex_unlock()
924 void rt_mutex_destroy(struct rt_mutex *lock)  in rt_mutex_destroy()  argument
926 	WARN_ON(rt_mutex_is_locked(lock));  in rt_mutex_destroy()
928 	lock->magic = NULL;  in rt_mutex_destroy()
943 void __rt_mutex_init(struct rt_mutex *lock, const char *name)  in __rt_mutex_init()  argument
945 	lock->owner = NULL;  in __rt_mutex_init()
946 	spin_lock_init(&lock->wait_lock);  in __rt_mutex_init()
947 	plist_head_init(&lock->wait_list, &lock->wait_lock);  in __rt_mutex_init()
949 	debug_rt_mutex_init(lock, name);  in __rt_mutex_init()
963 void rt_mutex_init_proxy_locked(struct rt_mutex *lock,  in rt_mutex_init_proxy_locked()  argument
966 	__rt_mutex_init(lock, NULL);  in rt_mutex_init_proxy_locked()
967 	debug_rt_mutex_proxy_lock(lock, proxy_owner);  in rt_mutex_init_proxy_locked()
968 	rt_mutex_set_owner(lock, proxy_owner, 0);  in rt_mutex_init_proxy_locked()
969 	rt_mutex_deadlock_account_lock(lock, proxy_owner);  in rt_mutex_init_proxy_locked()
980 void rt_mutex_proxy_unlock(struct rt_mutex *lock,  in rt_mutex_proxy_unlock()  argument
983 	debug_rt_mutex_proxy_unlock(lock);  in rt_mutex_proxy_unlock()
984 	rt_mutex_set_owner(lock, NULL, 0);  in rt_mutex_proxy_unlock()
1000 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)  in rt_mutex_next_owner()  argument
1002 	if (!rt_mutex_has_waiters(lock))  in rt_mutex_next_owner()
1005 	return rt_mutex_top_waiter(lock)->task;  in rt_mutex_next_owner()