Lines Matching refs:lock
53 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner) in rt_mutex_set_owner() argument
57 if (rt_mutex_has_waiters(lock)) in rt_mutex_set_owner()
60 lock->owner = (struct task_struct *)val; in rt_mutex_set_owner()
63 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock) in clear_rt_mutex_waiters() argument
65 lock->owner = (struct task_struct *) in clear_rt_mutex_waiters()
66 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS); in clear_rt_mutex_waiters()
69 static void fixup_rt_mutex_waiters(struct rt_mutex *lock) in fixup_rt_mutex_waiters() argument
71 unsigned long owner, *p = (unsigned long *) &lock->owner; in fixup_rt_mutex_waiters()
73 if (rt_mutex_has_waiters(lock)) in fixup_rt_mutex_waiters()
153 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) in mark_rt_mutex_waiters() argument
155 unsigned long owner, *p = (unsigned long *) &lock->owner; in mark_rt_mutex_waiters()
169 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, in unlock_rt_mutex_safe() argument
171 __releases(lock->wait_lock) in unlock_rt_mutex_safe()
173 struct task_struct *owner = rt_mutex_owner(lock); in unlock_rt_mutex_safe()
175 clear_rt_mutex_waiters(lock); in unlock_rt_mutex_safe()
176 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in unlock_rt_mutex_safe()
201 return rt_mutex_cmpxchg_release(lock, owner, NULL); in unlock_rt_mutex_safe()
209 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) in mark_rt_mutex_waiters() argument
211 lock->owner = (struct task_struct *) in mark_rt_mutex_waiters()
212 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); in mark_rt_mutex_waiters()
218 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, in unlock_rt_mutex_safe() argument
220 __releases(lock->wait_lock) in unlock_rt_mutex_safe()
222 lock->owner = NULL; in unlock_rt_mutex_safe()
223 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in unlock_rt_mutex_safe()
273 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) in rt_mutex_enqueue() argument
275 struct rb_node **link = &lock->waiters.rb_root.rb_node; in rt_mutex_enqueue()
292 rb_insert_color_cached(&waiter->tree_entry, &lock->waiters, leftmost); in rt_mutex_enqueue()
296 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) in rt_mutex_dequeue() argument
301 rb_erase_cached(&waiter->tree_entry, &lock->waiters); in rt_mutex_dequeue()
383 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; in task_blocked_on_lock()
459 struct rt_mutex *lock; in rt_mutex_adjust_prio_chain() local
538 if (next_lock != waiter->lock) in rt_mutex_adjust_prio_chain()
580 lock = waiter->lock; in rt_mutex_adjust_prio_chain()
586 if (!raw_spin_trylock(&lock->wait_lock)) { in rt_mutex_adjust_prio_chain()
601 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { in rt_mutex_adjust_prio_chain()
602 debug_rt_mutex_deadlock(chwalk, orig_waiter, lock); in rt_mutex_adjust_prio_chain()
603 raw_spin_unlock(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
625 if (!rt_mutex_owner(lock)) { in rt_mutex_adjust_prio_chain()
626 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
631 task = get_task_struct(rt_mutex_owner(lock)); in rt_mutex_adjust_prio_chain()
644 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
648 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
661 prerequeue_top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
664 rt_mutex_dequeue(lock, waiter); in rt_mutex_adjust_prio_chain()
685 rt_mutex_enqueue(lock, waiter); in rt_mutex_adjust_prio_chain()
698 if (!rt_mutex_owner(lock)) { in rt_mutex_adjust_prio_chain()
704 if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) in rt_mutex_adjust_prio_chain()
705 wake_up_process(rt_mutex_top_waiter(lock)->task); in rt_mutex_adjust_prio_chain()
706 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
711 task = get_task_struct(rt_mutex_owner(lock)); in rt_mutex_adjust_prio_chain()
715 if (waiter == rt_mutex_top_waiter(lock)) { in rt_mutex_adjust_prio_chain()
738 waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
763 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
767 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
807 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, in try_to_take_rt_mutex() argument
810 lockdep_assert_held(&lock->wait_lock); in try_to_take_rt_mutex()
829 mark_rt_mutex_waiters(lock); in try_to_take_rt_mutex()
834 if (rt_mutex_owner(lock)) in try_to_take_rt_mutex()
847 if (waiter != rt_mutex_top_waiter(lock)) in try_to_take_rt_mutex()
854 rt_mutex_dequeue(lock, waiter); in try_to_take_rt_mutex()
865 if (rt_mutex_has_waiters(lock)) { in try_to_take_rt_mutex()
872 rt_mutex_top_waiter(lock))) in try_to_take_rt_mutex()
904 if (rt_mutex_has_waiters(lock)) in try_to_take_rt_mutex()
905 rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock)); in try_to_take_rt_mutex()
910 debug_rt_mutex_lock(lock); in try_to_take_rt_mutex()
916 rt_mutex_set_owner(lock, task); in try_to_take_rt_mutex()
928 static int task_blocks_on_rt_mutex(struct rt_mutex *lock, in task_blocks_on_rt_mutex() argument
933 struct task_struct *owner = rt_mutex_owner(lock); in task_blocks_on_rt_mutex()
938 lockdep_assert_held(&lock->wait_lock); in task_blocks_on_rt_mutex()
954 waiter->lock = lock; in task_blocks_on_rt_mutex()
959 if (rt_mutex_has_waiters(lock)) in task_blocks_on_rt_mutex()
960 top_waiter = rt_mutex_top_waiter(lock); in task_blocks_on_rt_mutex()
961 rt_mutex_enqueue(lock, waiter); in task_blocks_on_rt_mutex()
971 if (waiter == rt_mutex_top_waiter(lock)) { in task_blocks_on_rt_mutex()
1001 raw_spin_unlock_irq(&lock->wait_lock); in task_blocks_on_rt_mutex()
1003 res = rt_mutex_adjust_prio_chain(owner, chwalk, lock, in task_blocks_on_rt_mutex()
1006 raw_spin_lock_irq(&lock->wait_lock); in task_blocks_on_rt_mutex()
1018 struct rt_mutex *lock) in mark_wakeup_next_waiter() argument
1024 waiter = rt_mutex_top_waiter(lock); in mark_wakeup_next_waiter()
1044 lock->owner = (void *) RT_MUTEX_HAS_WAITERS; in mark_wakeup_next_waiter()
1067 static void remove_waiter(struct rt_mutex *lock, in remove_waiter() argument
1070 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); in remove_waiter()
1071 struct task_struct *owner = rt_mutex_owner(lock); in remove_waiter()
1074 lockdep_assert_held(&lock->wait_lock); in remove_waiter()
1077 rt_mutex_dequeue(lock, waiter); in remove_waiter()
1092 if (rt_mutex_has_waiters(lock)) in remove_waiter()
1093 rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock)); in remove_waiter()
1112 raw_spin_unlock_irq(&lock->wait_lock); in remove_waiter()
1114 rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock, in remove_waiter()
1117 raw_spin_lock_irq(&lock->wait_lock); in remove_waiter()
1138 next_lock = waiter->lock; in rt_mutex_adjust_pi()
1167 __rt_mutex_slowlock(struct rt_mutex *lock, int state, in __rt_mutex_slowlock() argument
1175 if (try_to_take_rt_mutex(lock, current, waiter)) in __rt_mutex_slowlock()
1192 raw_spin_unlock_irq(&lock->wait_lock); in __rt_mutex_slowlock()
1198 raw_spin_lock_irq(&lock->wait_lock); in __rt_mutex_slowlock()
1230 rt_mutex_slowlock(struct rt_mutex *lock, int state, in rt_mutex_slowlock() argument
1248 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowlock()
1251 if (try_to_take_rt_mutex(lock, current, NULL)) { in rt_mutex_slowlock()
1252 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowlock()
1262 ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk); in rt_mutex_slowlock()
1266 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); in rt_mutex_slowlock()
1270 remove_waiter(lock, &waiter); in rt_mutex_slowlock()
1278 fixup_rt_mutex_waiters(lock); in rt_mutex_slowlock()
1280 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowlock()
1291 static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock) in __rt_mutex_slowtrylock() argument
1293 int ret = try_to_take_rt_mutex(lock, current, NULL); in __rt_mutex_slowtrylock()
1299 fixup_rt_mutex_waiters(lock); in __rt_mutex_slowtrylock()
1307 static inline int rt_mutex_slowtrylock(struct rt_mutex *lock) in rt_mutex_slowtrylock() argument
1317 if (rt_mutex_owner(lock)) in rt_mutex_slowtrylock()
1324 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowtrylock()
1326 ret = __rt_mutex_slowtrylock(lock); in rt_mutex_slowtrylock()
1328 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowtrylock()
1338 static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, in rt_mutex_slowunlock() argument
1344 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1346 debug_rt_mutex_unlock(lock); in rt_mutex_slowunlock()
1379 while (!rt_mutex_has_waiters(lock)) { in rt_mutex_slowunlock()
1381 if (unlock_rt_mutex_safe(lock, flags) == true) in rt_mutex_slowunlock()
1384 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1393 mark_wakeup_next_waiter(wake_q, lock); in rt_mutex_slowunlock()
1394 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1406 rt_mutex_fastlock(struct rt_mutex *lock, int state, in rt_mutex_fastlock() argument
1407 int (*slowfn)(struct rt_mutex *lock, int state, in rt_mutex_fastlock() argument
1411 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_fastlock()
1414 return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK); in rt_mutex_fastlock()
1418 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, in rt_mutex_timed_fastlock() argument
1421 int (*slowfn)(struct rt_mutex *lock, int state, in rt_mutex_timed_fastlock() argument
1426 likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_timed_fastlock()
1429 return slowfn(lock, state, timeout, chwalk); in rt_mutex_timed_fastlock()
1433 rt_mutex_fasttrylock(struct rt_mutex *lock, in rt_mutex_fasttrylock() argument
1434 int (*slowfn)(struct rt_mutex *lock)) in rt_mutex_fasttrylock() argument
1436 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_fasttrylock()
1439 return slowfn(lock); in rt_mutex_fasttrylock()
1454 rt_mutex_fastunlock(struct rt_mutex *lock, in rt_mutex_fastunlock() argument
1455 bool (*slowfn)(struct rt_mutex *lock, in rt_mutex_fastunlock() argument
1460 if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) in rt_mutex_fastunlock()
1463 if (slowfn(lock, &wake_q)) in rt_mutex_fastunlock()
1467 static inline void __rt_mutex_lock(struct rt_mutex *lock, unsigned int subclass) in __rt_mutex_lock() argument
1471 mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); in __rt_mutex_lock()
1472 rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock); in __rt_mutex_lock()
1482 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) in rt_mutex_lock_nested() argument
1484 __rt_mutex_lock(lock, subclass); in rt_mutex_lock_nested()
1495 void __sched rt_mutex_lock(struct rt_mutex *lock) in rt_mutex_lock() argument
1497 __rt_mutex_lock(lock, 0); in rt_mutex_lock()
1511 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock) in rt_mutex_lock_interruptible() argument
1517 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); in rt_mutex_lock_interruptible()
1518 ret = rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock); in rt_mutex_lock_interruptible()
1520 mutex_release(&lock->dep_map, 1, _RET_IP_); in rt_mutex_lock_interruptible()
1529 int __sched rt_mutex_futex_trylock(struct rt_mutex *lock) in rt_mutex_futex_trylock() argument
1531 return rt_mutex_slowtrylock(lock); in rt_mutex_futex_trylock()
1534 int __sched __rt_mutex_futex_trylock(struct rt_mutex *lock) in __rt_mutex_futex_trylock() argument
1536 return __rt_mutex_slowtrylock(lock); in __rt_mutex_futex_trylock()
1553 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout) in rt_mutex_timed_lock() argument
1559 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); in rt_mutex_timed_lock()
1560 ret = rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, in rt_mutex_timed_lock()
1564 mutex_release(&lock->dep_map, 1, _RET_IP_); in rt_mutex_timed_lock()
1581 int __sched rt_mutex_trylock(struct rt_mutex *lock) in rt_mutex_trylock() argument
1588 ret = rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock); in rt_mutex_trylock()
1590 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); in rt_mutex_trylock()
1601 void __sched rt_mutex_unlock(struct rt_mutex *lock) in rt_mutex_unlock() argument
1603 mutex_release(&lock->dep_map, 1, _RET_IP_); in rt_mutex_unlock()
1604 rt_mutex_fastunlock(lock, rt_mutex_slowunlock); in rt_mutex_unlock()
1612 bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock, in __rt_mutex_futex_unlock() argument
1615 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_futex_unlock()
1617 debug_rt_mutex_unlock(lock); in __rt_mutex_futex_unlock()
1619 if (!rt_mutex_has_waiters(lock)) { in __rt_mutex_futex_unlock()
1620 lock->owner = NULL; in __rt_mutex_futex_unlock()
1630 mark_wakeup_next_waiter(wake_q, lock); in __rt_mutex_futex_unlock()
1635 void __sched rt_mutex_futex_unlock(struct rt_mutex *lock) in rt_mutex_futex_unlock() argument
1641 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_futex_unlock()
1642 postunlock = __rt_mutex_futex_unlock(lock, &wake_q); in rt_mutex_futex_unlock()
1643 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_futex_unlock()
1657 void rt_mutex_destroy(struct rt_mutex *lock) in rt_mutex_destroy() argument
1659 WARN_ON(rt_mutex_is_locked(lock)); in rt_mutex_destroy()
1661 lock->magic = NULL; in rt_mutex_destroy()
1675 void __rt_mutex_init(struct rt_mutex *lock, const char *name, in __rt_mutex_init() argument
1678 lock->owner = NULL; in __rt_mutex_init()
1679 raw_spin_lock_init(&lock->wait_lock); in __rt_mutex_init()
1680 lock->waiters = RB_ROOT_CACHED; in __rt_mutex_init()
1683 debug_rt_mutex_init(lock, name, key); in __rt_mutex_init()
1701 void rt_mutex_init_proxy_locked(struct rt_mutex *lock, in rt_mutex_init_proxy_locked() argument
1704 __rt_mutex_init(lock, NULL, NULL); in rt_mutex_init_proxy_locked()
1705 debug_rt_mutex_proxy_lock(lock, proxy_owner); in rt_mutex_init_proxy_locked()
1706 rt_mutex_set_owner(lock, proxy_owner); in rt_mutex_init_proxy_locked()
1721 void rt_mutex_proxy_unlock(struct rt_mutex *lock, in rt_mutex_proxy_unlock() argument
1724 debug_rt_mutex_proxy_unlock(lock); in rt_mutex_proxy_unlock()
1725 rt_mutex_set_owner(lock, NULL); in rt_mutex_proxy_unlock()
1747 int __rt_mutex_start_proxy_lock(struct rt_mutex *lock, in __rt_mutex_start_proxy_lock() argument
1753 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_start_proxy_lock()
1755 if (try_to_take_rt_mutex(lock, task, NULL)) in __rt_mutex_start_proxy_lock()
1759 ret = task_blocks_on_rt_mutex(lock, waiter, task, in __rt_mutex_start_proxy_lock()
1762 if (ret && !rt_mutex_owner(lock)) { in __rt_mutex_start_proxy_lock()
1796 int rt_mutex_start_proxy_lock(struct rt_mutex *lock, in rt_mutex_start_proxy_lock() argument
1802 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_start_proxy_lock()
1803 ret = __rt_mutex_start_proxy_lock(lock, waiter, task); in rt_mutex_start_proxy_lock()
1805 remove_waiter(lock, waiter); in rt_mutex_start_proxy_lock()
1806 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_start_proxy_lock()
1823 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock) in rt_mutex_next_owner() argument
1825 if (!rt_mutex_has_waiters(lock)) in rt_mutex_next_owner()
1828 return rt_mutex_top_waiter(lock)->task; in rt_mutex_next_owner()
1848 int rt_mutex_wait_proxy_lock(struct rt_mutex *lock, in rt_mutex_wait_proxy_lock() argument
1854 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_wait_proxy_lock()
1857 ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter); in rt_mutex_wait_proxy_lock()
1862 fixup_rt_mutex_waiters(lock); in rt_mutex_wait_proxy_lock()
1863 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_wait_proxy_lock()
1888 bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock, in rt_mutex_cleanup_proxy_lock() argument
1893 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_cleanup_proxy_lock()
1905 try_to_take_rt_mutex(lock, current, waiter); in rt_mutex_cleanup_proxy_lock()
1910 if (rt_mutex_owner(lock) != current) { in rt_mutex_cleanup_proxy_lock()
1911 remove_waiter(lock, waiter); in rt_mutex_cleanup_proxy_lock()
1918 fixup_rt_mutex_waiters(lock); in rt_mutex_cleanup_proxy_lock()
1920 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_cleanup_proxy_lock()