Lines Matching refs:pi_state
131 struct futex_pi_state *pi_state; member
458 struct futex_pi_state *pi_state; in refill_pi_state_cache() local
463 pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL); in refill_pi_state_cache()
465 if (!pi_state) in refill_pi_state_cache()
468 INIT_LIST_HEAD(&pi_state->list); in refill_pi_state_cache()
470 pi_state->owner = NULL; in refill_pi_state_cache()
471 atomic_set(&pi_state->refcount, 1); in refill_pi_state_cache()
472 pi_state->key = FUTEX_KEY_INIT; in refill_pi_state_cache()
474 current->pi_state_cache = pi_state; in refill_pi_state_cache()
481 struct futex_pi_state *pi_state = current->pi_state_cache; in alloc_pi_state() local
483 WARN_ON(!pi_state); in alloc_pi_state()
486 return pi_state; in alloc_pi_state()
489 static void free_pi_state(struct futex_pi_state *pi_state) in free_pi_state() argument
491 if (!atomic_dec_and_test(&pi_state->refcount)) in free_pi_state()
498 if (pi_state->owner) { in free_pi_state()
499 raw_spin_lock_irq(&pi_state->owner->pi_lock); in free_pi_state()
500 list_del_init(&pi_state->list); in free_pi_state()
501 raw_spin_unlock_irq(&pi_state->owner->pi_lock); in free_pi_state()
503 rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner); in free_pi_state()
507 kfree(pi_state); in free_pi_state()
514 pi_state->owner = NULL; in free_pi_state()
515 atomic_set(&pi_state->refcount, 1); in free_pi_state()
516 current->pi_state_cache = pi_state; in free_pi_state()
546 struct futex_pi_state *pi_state; in exit_pi_state_list() local
561 pi_state = list_entry(next, struct futex_pi_state, list); in exit_pi_state_list()
562 key = pi_state->key; in exit_pi_state_list()
578 WARN_ON(pi_state->owner != curr); in exit_pi_state_list()
579 WARN_ON(list_empty(&pi_state->list)); in exit_pi_state_list()
580 list_del_init(&pi_state->list); in exit_pi_state_list()
581 pi_state->owner = NULL; in exit_pi_state_list()
584 rt_mutex_unlock(&pi_state->pi_mutex); in exit_pi_state_list()
646 struct futex_pi_state *pi_state = NULL; in lookup_pi_state() local
660 pi_state = this->pi_state; in lookup_pi_state()
665 if (unlikely(!pi_state)) in lookup_pi_state()
668 WARN_ON(!atomic_read(&pi_state->refcount)); in lookup_pi_state()
680 if (!pi_state->owner) { in lookup_pi_state()
712 if (!pi_state->owner) in lookup_pi_state()
722 if (pid != task_pid_vnr(pi_state->owner)) in lookup_pi_state()
726 atomic_inc(&pi_state->refcount); in lookup_pi_state()
727 *ps = pi_state; in lookup_pi_state()
765 pi_state = alloc_pi_state(); in lookup_pi_state()
771 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); in lookup_pi_state()
774 pi_state->key = *key; in lookup_pi_state()
776 WARN_ON(!list_empty(&pi_state->list)); in lookup_pi_state()
777 list_add(&pi_state->list, &p->pi_state_list); in lookup_pi_state()
778 pi_state->owner = p; in lookup_pi_state()
783 *ps = pi_state; in lookup_pi_state()
946 if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n")) in wake_futex()
975 struct futex_pi_state *pi_state = this->pi_state; in wake_futex_pi() local
979 if (!pi_state) in wake_futex_pi()
986 if (pi_state->owner != current) in wake_futex_pi()
989 raw_spin_lock(&pi_state->pi_mutex.wait_lock); in wake_futex_pi()
990 new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); in wake_futex_pi()
1012 raw_spin_unlock(&pi_state->pi_mutex.wait_lock); in wake_futex_pi()
1016 raw_spin_lock_irq(&pi_state->owner->pi_lock); in wake_futex_pi()
1017 WARN_ON(list_empty(&pi_state->list)); in wake_futex_pi()
1018 list_del_init(&pi_state->list); in wake_futex_pi()
1019 raw_spin_unlock_irq(&pi_state->owner->pi_lock); in wake_futex_pi()
1022 WARN_ON(!list_empty(&pi_state->list)); in wake_futex_pi()
1023 list_add(&pi_state->list, &new_owner->pi_state_list); in wake_futex_pi()
1024 pi_state->owner = new_owner; in wake_futex_pi()
1027 raw_spin_unlock(&pi_state->pi_mutex.wait_lock); in wake_futex_pi()
1028 rt_mutex_unlock(&pi_state->pi_mutex); in wake_futex_pi()
1098 if (this->pi_state || this->rt_waiter) { in futex_wake()
1181 if (this->pi_state || this->rt_waiter) { in futex_wake_op()
1197 if (this->pi_state || this->rt_waiter) { in futex_wake_op()
1363 struct futex_pi_state *pi_state = NULL; in futex_requeue() local
1398 if (pi_state != NULL) { in futex_requeue()
1403 free_pi_state(pi_state); in futex_requeue()
1404 pi_state = NULL; in futex_requeue()
1463 &key2, &pi_state, nr_requeue); in futex_requeue()
1472 WARN_ON(pi_state); in futex_requeue()
1478 &pi_state); in futex_requeue()
1521 this->pi_state) { in futex_requeue()
1548 atomic_inc(&pi_state->refcount); in futex_requeue()
1549 this->pi_state = pi_state; in futex_requeue()
1550 ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex, in futex_requeue()
1560 this->pi_state = NULL; in futex_requeue()
1561 free_pi_state(pi_state); in futex_requeue()
1586 if (pi_state != NULL) in futex_requeue()
1587 free_pi_state(pi_state); in futex_requeue()
1685 BUG_ON(q->pi_state); in unqueue_me()
1705 BUG_ON(!q->pi_state); in unqueue_me_pi()
1706 free_pi_state(q->pi_state); in unqueue_me_pi()
1707 q->pi_state = NULL; in unqueue_me_pi()
1722 struct futex_pi_state *pi_state = q->pi_state; in fixup_pi_state_owner() local
1723 struct task_struct *oldowner = pi_state->owner; in fixup_pi_state_owner()
1728 if (!pi_state->owner) in fixup_pi_state_owner()
1766 if (pi_state->owner != NULL) { in fixup_pi_state_owner()
1767 raw_spin_lock_irq(&pi_state->owner->pi_lock); in fixup_pi_state_owner()
1768 WARN_ON(list_empty(&pi_state->list)); in fixup_pi_state_owner()
1769 list_del_init(&pi_state->list); in fixup_pi_state_owner()
1770 raw_spin_unlock_irq(&pi_state->owner->pi_lock); in fixup_pi_state_owner()
1773 pi_state->owner = newowner; in fixup_pi_state_owner()
1776 WARN_ON(!list_empty(&pi_state->list)); in fixup_pi_state_owner()
1777 list_add(&pi_state->list, &newowner->pi_state_list); in fixup_pi_state_owner()
1801 if (pi_state->owner != oldowner) in fixup_pi_state_owner()
1837 if (q->pi_state->owner != current) in fixup_owner()
1846 if (q->pi_state->owner == current) { in fixup_owner()
1852 if (rt_mutex_trylock(&q->pi_state->pi_mutex)) { in fixup_owner()
1862 raw_spin_lock(&q->pi_state->pi_mutex.wait_lock); in fixup_owner()
1863 owner = rt_mutex_owner(&q->pi_state->pi_mutex); in fixup_owner()
1865 owner = rt_mutex_next_owner(&q->pi_state->pi_mutex); in fixup_owner()
1866 raw_spin_unlock(&q->pi_state->pi_mutex.wait_lock); in fixup_owner()
1875 if (rt_mutex_owner(&q->pi_state->pi_mutex) == current) in fixup_owner()
1878 q->pi_state->pi_mutex.owner, in fixup_owner()
1879 q->pi_state->owner); in fixup_owner()
2126 ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0); in futex_lock_pi()
2154 WARN_ON(!q.pi_state); in futex_lock_pi()
2159 ret = rt_mutex_timed_lock(&q.pi_state->pi_mutex, to, 1); in futex_lock_pi()
2161 ret = rt_mutex_trylock(&q.pi_state->pi_mutex); in futex_lock_pi()
2183 if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current)) in futex_lock_pi()
2184 rt_mutex_unlock(&q.pi_state->pi_mutex); in futex_lock_pi()
2477 if (q.pi_state && (q.pi_state->owner != current)) { in futex_wait_requeue_pi()
2488 WARN_ON(!q.pi_state); in futex_wait_requeue_pi()
2489 pi_mutex = &q.pi_state->pi_mutex; in futex_wait_requeue_pi()