Lines Matching refs:hb
319 static inline void hb_waiters_inc(struct futex_hash_bucket *hb) in hb_waiters_inc() argument
322 atomic_inc(&hb->waiters); in hb_waiters_inc()
334 static inline void hb_waiters_dec(struct futex_hash_bucket *hb) in hb_waiters_dec() argument
337 atomic_dec(&hb->waiters); in hb_waiters_dec()
341 static inline int hb_waiters_pending(struct futex_hash_bucket *hb) in hb_waiters_pending() argument
348 return atomic_read(&hb->waiters); in hb_waiters_pending()
695 static struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, in futex_top_waiter() argument
700 plist_for_each_entry(this, &hb->chain, list) { in futex_top_waiter()
845 struct futex_hash_bucket *hb; in exit_pi_state_list() local
860 hb = hash_futex(&key); in exit_pi_state_list()
880 spin_lock(&hb->lock); in exit_pi_state_list()
890 spin_unlock(&hb->lock); in exit_pi_state_list()
902 spin_unlock(&hb->lock); in exit_pi_state_list()
1304 struct futex_hash_bucket *hb, in lookup_pi_state() argument
1308 struct futex_q *top_waiter = futex_top_waiter(hb, key); in lookup_pi_state()
1364 static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb, in futex_lock_pi_atomic() argument
1398 top_waiter = futex_top_waiter(hb, key); in futex_lock_pi_atomic()
1450 struct futex_hash_bucket *hb; in __unqueue_futex() local
1456 hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock); in __unqueue_futex()
1457 plist_del(&q->list, &hb->chain); in __unqueue_futex()
1458 hb_waiters_dec(hb); in __unqueue_futex()
1592 struct futex_hash_bucket *hb; in futex_wake() local
1605 hb = hash_futex(&key); in futex_wake()
1608 if (!hb_waiters_pending(hb)) in futex_wake()
1611 spin_lock(&hb->lock); in futex_wake()
1613 plist_for_each_entry_safe(this, next, &hb->chain, list) { in futex_wake()
1630 spin_unlock(&hb->lock); in futex_wake()
1813 struct futex_hash_bucket *hb) in requeue_pi_wake_futex() argument
1822 q->lock_ptr = &hb->lock; in requeue_pi_wake_futex()
2195 __acquires(&hb->lock) in queue_lock()
2197 struct futex_hash_bucket *hb; in queue_lock() local
2199 hb = hash_futex(&q->key); in queue_lock()
2209 hb_waiters_inc(hb); /* implies smp_mb(); (A) */ in queue_lock()
2211 q->lock_ptr = &hb->lock; in queue_lock()
2213 spin_lock(&hb->lock); in queue_lock()
2214 return hb; in queue_lock()
2218 queue_unlock(struct futex_hash_bucket *hb) in queue_unlock() argument
2219 __releases(&hb->lock) in queue_unlock()
2221 spin_unlock(&hb->lock); in queue_unlock()
2222 hb_waiters_dec(hb); in queue_unlock()
2225 static inline void __queue_me(struct futex_q *q, struct futex_hash_bucket *hb) in __queue_me() argument
2240 plist_add(&q->list, &hb->chain); in __queue_me()
2256 static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) in queue_me() argument
2257 __releases(&hb->lock) in queue_me()
2259 __queue_me(q, hb); in queue_me()
2260 spin_unlock(&hb->lock); in queue_me()
2580 static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q, in futex_wait_queue_me() argument
2590 queue_me(q, hb); in futex_wait_queue_me()
2630 struct futex_q *q, struct futex_hash_bucket **hb) in futex_wait_setup() argument
2659 *hb = queue_lock(q); in futex_wait_setup()
2664 queue_unlock(*hb); in futex_wait_setup()
2677 queue_unlock(*hb); in futex_wait_setup()
2689 struct futex_hash_bucket *hb; in futex_wait() local
2704 ret = futex_wait_setup(uaddr, val, flags, &q, &hb); in futex_wait()
2709 futex_wait_queue_me(hb, &q, to); in futex_wait()
2780 struct futex_hash_bucket *hb; in futex_lock_pi() local
2798 hb = queue_lock(&q); in futex_lock_pi()
2800 ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, in futex_lock_pi()
2822 queue_unlock(hb); in futex_lock_pi()
2841 __queue_me(&q, hb); in futex_lock_pi()
2918 queue_unlock(hb); in futex_lock_pi()
2928 queue_unlock(hb); in futex_lock_pi()
2949 struct futex_hash_bucket *hb; in futex_unlock_pi() local
2969 hb = hash_futex(&key); in futex_unlock_pi()
2970 spin_lock(&hb->lock); in futex_unlock_pi()
2977 top_waiter = futex_top_waiter(hb, &key); in futex_unlock_pi()
3004 spin_unlock(&hb->lock); in futex_unlock_pi()
3043 spin_unlock(&hb->lock); in futex_unlock_pi()
3063 spin_unlock(&hb->lock); in futex_unlock_pi()
3097 int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, in handle_early_requeue_pi_wakeup() argument
3111 WARN_ON(q->lock_ptr && (&hb->lock != q->lock_ptr)); in handle_early_requeue_pi_wakeup()
3116 plist_del(&q->list, &hb->chain); in handle_early_requeue_pi_wakeup()
3117 hb_waiters_dec(hb); in handle_early_requeue_pi_wakeup()
3175 struct futex_hash_bucket *hb; in futex_wait_requeue_pi() local
3210 ret = futex_wait_setup(uaddr, val, flags, &q, &hb); in futex_wait_requeue_pi()
3219 queue_unlock(hb); in futex_wait_requeue_pi()
3225 futex_wait_queue_me(hb, &q, to); in futex_wait_requeue_pi()
3227 spin_lock(&hb->lock); in futex_wait_requeue_pi()
3228 ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); in futex_wait_requeue_pi()
3229 spin_unlock(&hb->lock); in futex_wait_requeue_pi()