Lines Matching refs:lock
54 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner) in rt_mutex_set_owner() argument
58 if (rt_mutex_has_waiters(lock)) in rt_mutex_set_owner()
61 WRITE_ONCE(lock->owner, (struct task_struct *)val); in rt_mutex_set_owner()
64 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock) in clear_rt_mutex_waiters() argument
66 lock->owner = (struct task_struct *) in clear_rt_mutex_waiters()
67 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS); in clear_rt_mutex_waiters()
70 static void fixup_rt_mutex_waiters(struct rt_mutex *lock) in fixup_rt_mutex_waiters() argument
72 unsigned long owner, *p = (unsigned long *) &lock->owner; in fixup_rt_mutex_waiters()
74 if (rt_mutex_has_waiters(lock)) in fixup_rt_mutex_waiters()
153 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) in mark_rt_mutex_waiters() argument
155 unsigned long owner, *p = (unsigned long *) &lock->owner; in mark_rt_mutex_waiters()
169 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, in unlock_rt_mutex_safe() argument
171 __releases(lock->wait_lock) in unlock_rt_mutex_safe()
173 struct task_struct *owner = rt_mutex_owner(lock); in unlock_rt_mutex_safe()
175 clear_rt_mutex_waiters(lock); in unlock_rt_mutex_safe()
176 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in unlock_rt_mutex_safe()
201 return rt_mutex_cmpxchg_release(lock, owner, NULL); in unlock_rt_mutex_safe()
208 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) in mark_rt_mutex_waiters() argument
210 lock->owner = (struct task_struct *) in mark_rt_mutex_waiters()
211 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); in mark_rt_mutex_waiters()
217 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, in unlock_rt_mutex_safe() argument
219 __releases(lock->wait_lock) in unlock_rt_mutex_safe()
221 lock->owner = NULL; in unlock_rt_mutex_safe()
222 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in unlock_rt_mutex_safe()
272 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) in rt_mutex_enqueue() argument
274 struct rb_node **link = &lock->waiters.rb_root.rb_node; in rt_mutex_enqueue()
291 rb_insert_color_cached(&waiter->tree_entry, &lock->waiters, leftmost); in rt_mutex_enqueue()
295 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) in rt_mutex_dequeue() argument
300 rb_erase_cached(&waiter->tree_entry, &lock->waiters); in rt_mutex_dequeue()
382 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; in task_blocked_on_lock()
458 struct rt_mutex *lock; in rt_mutex_adjust_prio_chain() local
537 if (next_lock != waiter->lock) in rt_mutex_adjust_prio_chain()
579 lock = waiter->lock; in rt_mutex_adjust_prio_chain()
585 if (!raw_spin_trylock(&lock->wait_lock)) { in rt_mutex_adjust_prio_chain()
600 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { in rt_mutex_adjust_prio_chain()
601 debug_rt_mutex_deadlock(chwalk, orig_waiter, lock); in rt_mutex_adjust_prio_chain()
602 raw_spin_unlock(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
624 if (!rt_mutex_owner(lock)) { in rt_mutex_adjust_prio_chain()
625 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
630 task = get_task_struct(rt_mutex_owner(lock)); in rt_mutex_adjust_prio_chain()
643 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
647 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
660 prerequeue_top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
663 rt_mutex_dequeue(lock, waiter); in rt_mutex_adjust_prio_chain()
684 rt_mutex_enqueue(lock, waiter); in rt_mutex_adjust_prio_chain()
697 if (!rt_mutex_owner(lock)) { in rt_mutex_adjust_prio_chain()
703 if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) in rt_mutex_adjust_prio_chain()
704 wake_up_process(rt_mutex_top_waiter(lock)->task); in rt_mutex_adjust_prio_chain()
705 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
710 task = get_task_struct(rt_mutex_owner(lock)); in rt_mutex_adjust_prio_chain()
714 if (waiter == rt_mutex_top_waiter(lock)) { in rt_mutex_adjust_prio_chain()
737 waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
762 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
766 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
806 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, in try_to_take_rt_mutex() argument
809 lockdep_assert_held(&lock->wait_lock); in try_to_take_rt_mutex()
828 mark_rt_mutex_waiters(lock); in try_to_take_rt_mutex()
833 if (rt_mutex_owner(lock)) in try_to_take_rt_mutex()
846 if (waiter != rt_mutex_top_waiter(lock)) in try_to_take_rt_mutex()
853 rt_mutex_dequeue(lock, waiter); in try_to_take_rt_mutex()
864 if (rt_mutex_has_waiters(lock)) { in try_to_take_rt_mutex()
871 rt_mutex_top_waiter(lock))) in try_to_take_rt_mutex()
903 if (rt_mutex_has_waiters(lock)) in try_to_take_rt_mutex()
904 rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock)); in try_to_take_rt_mutex()
909 debug_rt_mutex_lock(lock); in try_to_take_rt_mutex()
915 rt_mutex_set_owner(lock, task); in try_to_take_rt_mutex()
927 static int task_blocks_on_rt_mutex(struct rt_mutex *lock, in task_blocks_on_rt_mutex() argument
932 struct task_struct *owner = rt_mutex_owner(lock); in task_blocks_on_rt_mutex()
937 lockdep_assert_held(&lock->wait_lock); in task_blocks_on_rt_mutex()
953 waiter->lock = lock; in task_blocks_on_rt_mutex()
958 if (rt_mutex_has_waiters(lock)) in task_blocks_on_rt_mutex()
959 top_waiter = rt_mutex_top_waiter(lock); in task_blocks_on_rt_mutex()
960 rt_mutex_enqueue(lock, waiter); in task_blocks_on_rt_mutex()
970 if (waiter == rt_mutex_top_waiter(lock)) { in task_blocks_on_rt_mutex()
1000 raw_spin_unlock_irq(&lock->wait_lock); in task_blocks_on_rt_mutex()
1002 res = rt_mutex_adjust_prio_chain(owner, chwalk, lock, in task_blocks_on_rt_mutex()
1005 raw_spin_lock_irq(&lock->wait_lock); in task_blocks_on_rt_mutex()
1017 struct rt_mutex *lock) in mark_wakeup_next_waiter() argument
1023 waiter = rt_mutex_top_waiter(lock); in mark_wakeup_next_waiter()
1043 lock->owner = (void *) RT_MUTEX_HAS_WAITERS; in mark_wakeup_next_waiter()
1066 static void remove_waiter(struct rt_mutex *lock, in remove_waiter() argument
1069 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); in remove_waiter()
1070 struct task_struct *owner = rt_mutex_owner(lock); in remove_waiter()
1073 lockdep_assert_held(&lock->wait_lock); in remove_waiter()
1076 rt_mutex_dequeue(lock, waiter); in remove_waiter()
1091 if (rt_mutex_has_waiters(lock)) in remove_waiter()
1092 rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock)); in remove_waiter()
1111 raw_spin_unlock_irq(&lock->wait_lock); in remove_waiter()
1113 rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock, in remove_waiter()
1116 raw_spin_lock_irq(&lock->wait_lock); in remove_waiter()
1137 next_lock = waiter->lock; in rt_mutex_adjust_pi()
1166 __rt_mutex_slowlock(struct rt_mutex *lock, int state, in __rt_mutex_slowlock() argument
1172 trace_android_vh_rtmutex_wait_start(lock); in __rt_mutex_slowlock()
1175 if (try_to_take_rt_mutex(lock, current, waiter)) in __rt_mutex_slowlock()
1192 raw_spin_unlock_irq(&lock->wait_lock); in __rt_mutex_slowlock()
1198 raw_spin_lock_irq(&lock->wait_lock); in __rt_mutex_slowlock()
1202 trace_android_vh_rtmutex_wait_finish(lock); in __rt_mutex_slowlock()
1231 rt_mutex_slowlock(struct rt_mutex *lock, int state, in rt_mutex_slowlock() argument
1249 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowlock()
1252 if (try_to_take_rt_mutex(lock, current, NULL)) { in rt_mutex_slowlock()
1253 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowlock()
1263 ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk); in rt_mutex_slowlock()
1267 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); in rt_mutex_slowlock()
1271 remove_waiter(lock, &waiter); in rt_mutex_slowlock()
1279 fixup_rt_mutex_waiters(lock); in rt_mutex_slowlock()
1281 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowlock()
1292 static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock) in __rt_mutex_slowtrylock() argument
1294 int ret = try_to_take_rt_mutex(lock, current, NULL); in __rt_mutex_slowtrylock()
1300 fixup_rt_mutex_waiters(lock); in __rt_mutex_slowtrylock()
1308 static inline int rt_mutex_slowtrylock(struct rt_mutex *lock) in rt_mutex_slowtrylock() argument
1318 if (rt_mutex_owner(lock)) in rt_mutex_slowtrylock()
1325 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowtrylock()
1327 ret = __rt_mutex_slowtrylock(lock); in rt_mutex_slowtrylock()
1329 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowtrylock()
1339 static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, in rt_mutex_slowunlock() argument
1345 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1347 debug_rt_mutex_unlock(lock); in rt_mutex_slowunlock()
1380 while (!rt_mutex_has_waiters(lock)) { in rt_mutex_slowunlock()
1382 if (unlock_rt_mutex_safe(lock, flags) == true) in rt_mutex_slowunlock()
1385 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1394 mark_wakeup_next_waiter(wake_q, lock); in rt_mutex_slowunlock()
1395 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1407 rt_mutex_fastlock(struct rt_mutex *lock, int state, in rt_mutex_fastlock() argument
1408 int (*slowfn)(struct rt_mutex *lock, int state, in rt_mutex_fastlock() argument
1412 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_fastlock()
1415 return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK); in rt_mutex_fastlock()
1419 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, in rt_mutex_timed_fastlock() argument
1422 int (*slowfn)(struct rt_mutex *lock, int state, in rt_mutex_timed_fastlock() argument
1427 likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_timed_fastlock()
1430 return slowfn(lock, state, timeout, chwalk); in rt_mutex_timed_fastlock()
1434 rt_mutex_fasttrylock(struct rt_mutex *lock, in rt_mutex_fasttrylock() argument
1435 int (*slowfn)(struct rt_mutex *lock)) in rt_mutex_fasttrylock() argument
1437 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_fasttrylock()
1440 return slowfn(lock); in rt_mutex_fasttrylock()
1455 rt_mutex_fastunlock(struct rt_mutex *lock, in rt_mutex_fastunlock() argument
1456 bool (*slowfn)(struct rt_mutex *lock, in rt_mutex_fastunlock() argument
1461 if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) in rt_mutex_fastunlock()
1464 if (slowfn(lock, &wake_q)) in rt_mutex_fastunlock()
1468 static inline void __rt_mutex_lock(struct rt_mutex *lock, unsigned int subclass) in __rt_mutex_lock() argument
1472 mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); in __rt_mutex_lock()
1473 rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock); in __rt_mutex_lock()
1484 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) in rt_mutex_lock_nested() argument
1486 __rt_mutex_lock(lock, subclass); in rt_mutex_lock_nested()
1497 void __sched rt_mutex_lock(struct rt_mutex *lock) in rt_mutex_lock() argument
1499 __rt_mutex_lock(lock, 0); in rt_mutex_lock()
1513 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock) in rt_mutex_lock_interruptible() argument
1519 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); in rt_mutex_lock_interruptible()
1520 ret = rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock); in rt_mutex_lock_interruptible()
1522 mutex_release(&lock->dep_map, _RET_IP_); in rt_mutex_lock_interruptible()
1533 int __sched rt_mutex_futex_trylock(struct rt_mutex *lock) in rt_mutex_futex_trylock() argument
1535 return rt_mutex_slowtrylock(lock); in rt_mutex_futex_trylock()
1538 int __sched __rt_mutex_futex_trylock(struct rt_mutex *lock) in __rt_mutex_futex_trylock() argument
1540 return __rt_mutex_slowtrylock(lock); in __rt_mutex_futex_trylock()
1557 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout) in rt_mutex_timed_lock() argument
1563 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); in rt_mutex_timed_lock()
1564 ret = rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, in rt_mutex_timed_lock()
1568 mutex_release(&lock->dep_map, _RET_IP_); in rt_mutex_timed_lock()
1587 int __sched rt_mutex_trylock(struct rt_mutex *lock) in rt_mutex_trylock() argument
1594 ret = rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock); in rt_mutex_trylock()
1596 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); in rt_mutex_trylock()
1609 void __sched rt_mutex_unlock(struct rt_mutex *lock) in rt_mutex_unlock() argument
1611 mutex_release(&lock->dep_map, _RET_IP_); in rt_mutex_unlock()
1612 rt_mutex_fastunlock(lock, rt_mutex_slowunlock); in rt_mutex_unlock()
1621 bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock, in __rt_mutex_futex_unlock() argument
1624 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_futex_unlock()
1626 debug_rt_mutex_unlock(lock); in __rt_mutex_futex_unlock()
1628 if (!rt_mutex_has_waiters(lock)) { in __rt_mutex_futex_unlock()
1629 lock->owner = NULL; in __rt_mutex_futex_unlock()
1639 mark_wakeup_next_waiter(wake_q, lock); in __rt_mutex_futex_unlock()
1644 void __sched rt_mutex_futex_unlock(struct rt_mutex *lock) in rt_mutex_futex_unlock() argument
1650 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_futex_unlock()
1651 postunlock = __rt_mutex_futex_unlock(lock, &wake_q); in rt_mutex_futex_unlock()
1652 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_futex_unlock()
1666 void rt_mutex_destroy(struct rt_mutex *lock) in rt_mutex_destroy() argument
1668 WARN_ON(rt_mutex_is_locked(lock)); in rt_mutex_destroy()
1670 lock->magic = NULL; in rt_mutex_destroy()
1684 void __rt_mutex_init(struct rt_mutex *lock, const char *name, in __rt_mutex_init() argument
1687 lock->owner = NULL; in __rt_mutex_init()
1688 raw_spin_lock_init(&lock->wait_lock); in __rt_mutex_init()
1689 lock->waiters = RB_ROOT_CACHED; in __rt_mutex_init()
1692 debug_rt_mutex_init(lock, name, key); in __rt_mutex_init()
1710 void rt_mutex_init_proxy_locked(struct rt_mutex *lock, in rt_mutex_init_proxy_locked() argument
1713 __rt_mutex_init(lock, NULL, NULL); in rt_mutex_init_proxy_locked()
1714 debug_rt_mutex_proxy_lock(lock, proxy_owner); in rt_mutex_init_proxy_locked()
1715 rt_mutex_set_owner(lock, proxy_owner); in rt_mutex_init_proxy_locked()
1730 void rt_mutex_proxy_unlock(struct rt_mutex *lock) in rt_mutex_proxy_unlock() argument
1732 debug_rt_mutex_proxy_unlock(lock); in rt_mutex_proxy_unlock()
1733 rt_mutex_set_owner(lock, NULL); in rt_mutex_proxy_unlock()
1755 int __rt_mutex_start_proxy_lock(struct rt_mutex *lock, in __rt_mutex_start_proxy_lock() argument
1761 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_start_proxy_lock()
1763 if (try_to_take_rt_mutex(lock, task, NULL)) in __rt_mutex_start_proxy_lock()
1767 ret = task_blocks_on_rt_mutex(lock, waiter, task, in __rt_mutex_start_proxy_lock()
1770 if (ret && !rt_mutex_owner(lock)) { in __rt_mutex_start_proxy_lock()
1804 int rt_mutex_start_proxy_lock(struct rt_mutex *lock, in rt_mutex_start_proxy_lock() argument
1810 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_start_proxy_lock()
1811 ret = __rt_mutex_start_proxy_lock(lock, waiter, task); in rt_mutex_start_proxy_lock()
1813 remove_waiter(lock, waiter); in rt_mutex_start_proxy_lock()
1814 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_start_proxy_lock()
1831 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock) in rt_mutex_next_owner() argument
1833 if (!rt_mutex_has_waiters(lock)) in rt_mutex_next_owner()
1836 return rt_mutex_top_waiter(lock)->task; in rt_mutex_next_owner()
1856 int rt_mutex_wait_proxy_lock(struct rt_mutex *lock, in rt_mutex_wait_proxy_lock() argument
1862 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_wait_proxy_lock()
1865 ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter); in rt_mutex_wait_proxy_lock()
1870 fixup_rt_mutex_waiters(lock); in rt_mutex_wait_proxy_lock()
1871 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_wait_proxy_lock()
1896 bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock, in rt_mutex_cleanup_proxy_lock() argument
1901 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_cleanup_proxy_lock()
1913 try_to_take_rt_mutex(lock, current, waiter); in rt_mutex_cleanup_proxy_lock()
1918 if (rt_mutex_owner(lock) != current) { in rt_mutex_cleanup_proxy_lock()
1919 remove_waiter(lock, waiter); in rt_mutex_cleanup_proxy_lock()
1926 fixup_rt_mutex_waiters(lock); in rt_mutex_cleanup_proxy_lock()
1928 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_cleanup_proxy_lock()