Lines Matching refs:pi_state
219 struct futex_pi_state *pi_state; member
747 struct futex_pi_state *pi_state; in refill_pi_state_cache() local
752 pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL); in refill_pi_state_cache()
754 if (!pi_state) in refill_pi_state_cache()
757 INIT_LIST_HEAD(&pi_state->list); in refill_pi_state_cache()
759 pi_state->owner = NULL; in refill_pi_state_cache()
760 refcount_set(&pi_state->refcount, 1); in refill_pi_state_cache()
761 pi_state->key = FUTEX_KEY_INIT; in refill_pi_state_cache()
763 current->pi_state_cache = pi_state; in refill_pi_state_cache()
770 struct futex_pi_state *pi_state = current->pi_state_cache; in alloc_pi_state() local
772 WARN_ON(!pi_state); in alloc_pi_state()
775 return pi_state; in alloc_pi_state()
778 static void pi_state_update_owner(struct futex_pi_state *pi_state, in pi_state_update_owner() argument
781 struct task_struct *old_owner = pi_state->owner; in pi_state_update_owner()
783 lockdep_assert_held(&pi_state->pi_mutex.wait_lock); in pi_state_update_owner()
787 WARN_ON(list_empty(&pi_state->list)); in pi_state_update_owner()
788 list_del_init(&pi_state->list); in pi_state_update_owner()
794 WARN_ON(!list_empty(&pi_state->list)); in pi_state_update_owner()
795 list_add(&pi_state->list, &new_owner->pi_state_list); in pi_state_update_owner()
796 pi_state->owner = new_owner; in pi_state_update_owner()
801 static void get_pi_state(struct futex_pi_state *pi_state) in get_pi_state() argument
803 WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount)); in get_pi_state()
810 static void put_pi_state(struct futex_pi_state *pi_state) in put_pi_state() argument
812 if (!pi_state) in put_pi_state()
815 if (!refcount_dec_and_test(&pi_state->refcount)) in put_pi_state()
822 if (pi_state->owner) { in put_pi_state()
825 raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags); in put_pi_state()
826 pi_state_update_owner(pi_state, NULL); in put_pi_state()
827 rt_mutex_proxy_unlock(&pi_state->pi_mutex); in put_pi_state()
828 raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags); in put_pi_state()
832 kfree(pi_state); in put_pi_state()
839 pi_state->owner = NULL; in put_pi_state()
840 refcount_set(&pi_state->refcount, 1); in put_pi_state()
841 current->pi_state_cache = pi_state; in put_pi_state()
855 struct futex_pi_state *pi_state; in exit_pi_state_list() local
869 pi_state = list_entry(next, struct futex_pi_state, list); in exit_pi_state_list()
870 key = pi_state->key; in exit_pi_state_list()
883 if (!refcount_inc_not_zero(&pi_state->refcount)) { in exit_pi_state_list()
892 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in exit_pi_state_list()
900 raw_spin_unlock(&pi_state->pi_mutex.wait_lock); in exit_pi_state_list()
902 put_pi_state(pi_state); in exit_pi_state_list()
906 WARN_ON(pi_state->owner != curr); in exit_pi_state_list()
907 WARN_ON(list_empty(&pi_state->list)); in exit_pi_state_list()
908 list_del_init(&pi_state->list); in exit_pi_state_list()
909 pi_state->owner = NULL; in exit_pi_state_list()
912 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in exit_pi_state_list()
915 rt_mutex_futex_unlock(&pi_state->pi_mutex); in exit_pi_state_list()
916 put_pi_state(pi_state); in exit_pi_state_list()
1016 struct futex_pi_state *pi_state, in attach_to_pi_state() argument
1026 if (unlikely(!pi_state)) in attach_to_pi_state()
1041 WARN_ON(!refcount_read(&pi_state->refcount)); in attach_to_pi_state()
1047 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
1070 if (!pi_state->owner) { in attach_to_pi_state()
1098 if (!pi_state->owner) in attach_to_pi_state()
1107 if (pid != task_pid_vnr(pi_state->owner)) in attach_to_pi_state()
1111 get_pi_state(pi_state); in attach_to_pi_state()
1112 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
1113 *ps = pi_state; in attach_to_pi_state()
1129 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
1229 struct futex_pi_state *pi_state; in attach_to_pi_owner() local
1287 pi_state = alloc_pi_state(); in attach_to_pi_owner()
1293 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); in attach_to_pi_owner()
1296 pi_state->key = *key; in attach_to_pi_owner()
1298 WARN_ON(!list_empty(&pi_state->list)); in attach_to_pi_owner()
1299 list_add(&pi_state->list, &p->pi_state_list); in attach_to_pi_owner()
1304 pi_state->owner = p; in attach_to_pi_owner()
1309 *ps = pi_state; in attach_to_pi_owner()
1326 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps); in lookup_pi_state()
1411 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps); in futex_lock_pi_atomic()
1482 if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n")) in mark_wake_futex()
1506 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state) in wake_futex_pi() argument
1514 new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); in wake_futex_pi()
1560 pi_state_update_owner(pi_state, new_owner); in wake_futex_pi()
1561 postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q); in wake_futex_pi()
1565 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in wake_futex_pi()
1628 if (this->pi_state || this->rt_waiter) { in futex_wake()
1754 if (this->pi_state || this->rt_waiter) { in futex_wake_op()
1768 if (this->pi_state || this->rt_waiter) { in futex_wake_op()
1941 struct futex_pi_state *pi_state = NULL; in futex_requeue() local
2043 &key2, &pi_state, in futex_requeue()
2056 WARN_ON(pi_state); in futex_requeue()
2071 &pi_state, &exiting); in futex_requeue()
2126 this->pi_state) { in futex_requeue()
2157 get_pi_state(pi_state); in futex_requeue()
2158 this->pi_state = pi_state; in futex_requeue()
2159 ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex, in futex_requeue()
2182 this->pi_state = NULL; in futex_requeue()
2183 put_pi_state(pi_state); in futex_requeue()
2199 put_pi_state(pi_state); in futex_requeue()
2326 BUG_ON(q->pi_state); in unqueue_me()
2345 BUG_ON(!q->pi_state); in unqueue_me_pi()
2346 put_pi_state(q->pi_state); in unqueue_me_pi()
2347 q->pi_state = NULL; in unqueue_me_pi()
2355 struct futex_pi_state *pi_state = q->pi_state; in __fixup_pi_state_owner() local
2360 oldowner = pi_state->owner; in __fixup_pi_state_owner()
2395 if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) { in __fixup_pi_state_owner()
2404 newowner = rt_mutex_owner(&pi_state->pi_mutex); in __fixup_pi_state_owner()
2431 if (!pi_state->owner) in __fixup_pi_state_owner()
2454 pi_state_update_owner(pi_state, newowner); in __fixup_pi_state_owner()
2472 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in __fixup_pi_state_owner()
2491 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in __fixup_pi_state_owner()
2496 if (pi_state->owner != oldowner) in __fixup_pi_state_owner()
2519 pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex)); in __fixup_pi_state_owner()
2527 struct futex_pi_state *pi_state = q->pi_state; in fixup_pi_state_owner() local
2532 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in fixup_pi_state_owner()
2534 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in fixup_pi_state_owner()
2566 if (q->pi_state->owner != current) in fixup_owner()
2579 if (q->pi_state->owner == current) in fixup_owner()
2586 if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current)) in fixup_owner()
2822 ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, in futex_lock_pi()
2858 WARN_ON(!q.pi_state); in futex_lock_pi()
2866 ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex); in futex_lock_pi()
2887 raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock); in futex_lock_pi()
2894 ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current); in futex_lock_pi()
2895 raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock); in futex_lock_pi()
2906 ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter); in futex_lock_pi()
2919 if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter)) in futex_lock_pi()
3001 struct futex_pi_state *pi_state = top_waiter->pi_state; in futex_unlock_pi() local
3004 if (!pi_state) in futex_unlock_pi()
3011 if (pi_state->owner != current) in futex_unlock_pi()
3014 get_pi_state(pi_state); in futex_unlock_pi()
3025 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in futex_unlock_pi()
3029 ret = wake_futex_pi(uaddr, uval, pi_state); in futex_unlock_pi()
3031 put_pi_state(pi_state); in futex_unlock_pi()
3270 if (q.pi_state && (q.pi_state->owner != current)) { in futex_wait_requeue_pi()
3277 put_pi_state(q.pi_state); in futex_wait_requeue_pi()
3293 WARN_ON(!q.pi_state); in futex_wait_requeue_pi()
3294 pi_mutex = &q.pi_state->pi_mutex; in futex_wait_requeue_pi()