Lines Matching refs:pi_state
14 struct futex_pi_state *pi_state; in refill_pi_state_cache() local
19 pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL); in refill_pi_state_cache()
21 if (!pi_state) in refill_pi_state_cache()
24 INIT_LIST_HEAD(&pi_state->list); in refill_pi_state_cache()
26 pi_state->owner = NULL; in refill_pi_state_cache()
27 refcount_set(&pi_state->refcount, 1); in refill_pi_state_cache()
28 pi_state->key = FUTEX_KEY_INIT; in refill_pi_state_cache()
30 current->pi_state_cache = pi_state; in refill_pi_state_cache()
37 struct futex_pi_state *pi_state = current->pi_state_cache; in alloc_pi_state() local
39 WARN_ON(!pi_state); in alloc_pi_state()
42 return pi_state; in alloc_pi_state()
45 static void pi_state_update_owner(struct futex_pi_state *pi_state, in pi_state_update_owner() argument
48 struct task_struct *old_owner = pi_state->owner; in pi_state_update_owner()
50 lockdep_assert_held(&pi_state->pi_mutex.wait_lock); in pi_state_update_owner()
54 WARN_ON(list_empty(&pi_state->list)); in pi_state_update_owner()
55 list_del_init(&pi_state->list); in pi_state_update_owner()
61 WARN_ON(!list_empty(&pi_state->list)); in pi_state_update_owner()
62 list_add(&pi_state->list, &new_owner->pi_state_list); in pi_state_update_owner()
63 pi_state->owner = new_owner; in pi_state_update_owner()
68 void get_pi_state(struct futex_pi_state *pi_state) in get_pi_state() argument
70 WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount)); in get_pi_state()
77 void put_pi_state(struct futex_pi_state *pi_state) in put_pi_state() argument
79 if (!pi_state) in put_pi_state()
82 if (!refcount_dec_and_test(&pi_state->refcount)) in put_pi_state()
89 if (pi_state->owner) { in put_pi_state()
92 raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags); in put_pi_state()
93 pi_state_update_owner(pi_state, NULL); in put_pi_state()
94 rt_mutex_proxy_unlock(&pi_state->pi_mutex); in put_pi_state()
95 raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags); in put_pi_state()
99 kfree(pi_state); in put_pi_state()
106 pi_state->owner = NULL; in put_pi_state()
107 refcount_set(&pi_state->refcount, 1); in put_pi_state()
108 current->pi_state_cache = pi_state; in put_pi_state()
203 struct futex_pi_state *pi_state, in attach_to_pi_state() argument
213 if (unlikely(!pi_state)) in attach_to_pi_state()
228 WARN_ON(!refcount_read(&pi_state->refcount)); in attach_to_pi_state()
234 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
257 if (!pi_state->owner) { in attach_to_pi_state()
285 if (!pi_state->owner) in attach_to_pi_state()
294 if (pid != task_pid_vnr(pi_state->owner)) in attach_to_pi_state()
298 get_pi_state(pi_state); in attach_to_pi_state()
299 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
300 *ps = pi_state; in attach_to_pi_state()
316 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
385 struct futex_pi_state *pi_state = alloc_pi_state(); in __attach_to_pi_owner() local
391 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); in __attach_to_pi_owner()
394 pi_state->key = *key; in __attach_to_pi_owner()
396 WARN_ON(!list_empty(&pi_state->list)); in __attach_to_pi_owner()
397 list_add(&pi_state->list, &p->pi_state_list); in __attach_to_pi_owner()
402 pi_state->owner = p; in __attach_to_pi_owner()
404 *ps = pi_state; in __attach_to_pi_owner()
550 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps); in futex_lock_pi_atomic()
613 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state) in wake_futex_pi() argument
622 top_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex); in wake_futex_pi()
670 pi_state_update_owner(pi_state, new_owner); in wake_futex_pi()
671 postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wqh); in wake_futex_pi()
675 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in wake_futex_pi()
686 struct futex_pi_state *pi_state = q->pi_state; in __fixup_pi_state_owner() local
691 oldowner = pi_state->owner; in __fixup_pi_state_owner()
726 if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) { in __fixup_pi_state_owner()
735 newowner = rt_mutex_owner(&pi_state->pi_mutex); in __fixup_pi_state_owner()
762 if (!pi_state->owner) in __fixup_pi_state_owner()
785 pi_state_update_owner(pi_state, newowner); in __fixup_pi_state_owner()
803 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in __fixup_pi_state_owner()
822 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in __fixup_pi_state_owner()
827 if (pi_state->owner != oldowner) in __fixup_pi_state_owner()
850 pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex)); in __fixup_pi_state_owner()
858 struct futex_pi_state *pi_state = q->pi_state; in fixup_pi_state_owner() local
863 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in fixup_pi_state_owner()
865 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in fixup_pi_state_owner()
895 if (q->pi_state->owner != current) in fixup_pi_owner()
908 if (q->pi_state->owner == current) in fixup_pi_owner()
915 if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current)) in fixup_pi_owner()
955 ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, in futex_lock_pi()
991 WARN_ON(!q.pi_state); in futex_lock_pi()
999 ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex); in futex_lock_pi()
1020 raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock); in futex_lock_pi()
1027 ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current); in futex_lock_pi()
1028 raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock); in futex_lock_pi()
1039 ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter); in futex_lock_pi()
1052 if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter)) in futex_lock_pi()
1134 struct futex_pi_state *pi_state = top_waiter->pi_state; in futex_unlock_pi() local
1137 if (!pi_state) in futex_unlock_pi()
1144 if (pi_state->owner != current) in futex_unlock_pi()
1147 get_pi_state(pi_state); in futex_unlock_pi()
1158 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in futex_unlock_pi()
1162 ret = wake_futex_pi(uaddr, uval, pi_state); in futex_unlock_pi()
1164 put_pi_state(pi_state); in futex_unlock_pi()