Lines Matching refs:val
104 static inline int decode_tail_cpu(u32 val) in decode_tail_cpu() argument
106 return (val >> _Q_TAIL_CPU_OFFSET) - 1; in decode_tail_cpu()
109 static inline int get_owner_cpu(u32 val) in get_owner_cpu() argument
111 return (val & _Q_OWNER_CPU_MASK) >> _Q_OWNER_CPU_OFFSET; in get_owner_cpu()
144 : "r" (&lock->val), "r"(tail), "r" (newval), in trylock_clean_tail()
174 : "r" (&lock->val), "r" (tail), "r"(_Q_TAIL_CPU_MASK) in publish_tail_cpu()
190 : "r" (&lock->val), "r" (_Q_MUST_Q_VAL) in set_mustq()
206 : "r" (&lock->val), "r" (_Q_MUST_Q_VAL) in clear_mustq()
228 : "r" (&lock->val), "r"(old), "r" (new) in try_set_sleepy()
234 static __always_inline void seen_sleepy_owner(struct qspinlock *lock, u32 val) in seen_sleepy_owner() argument
239 if (!(val & _Q_SLEEPY_VAL)) in seen_sleepy_owner()
240 try_set_sleepy(lock, val); in seen_sleepy_owner()
250 static __always_inline void seen_sleepy_node(struct qspinlock *lock, u32 val) in seen_sleepy_node() argument
255 if (val & _Q_LOCKED_VAL) { in seen_sleepy_node()
256 if (!(val & _Q_SLEEPY_VAL)) in seen_sleepy_node()
257 try_set_sleepy(lock, val); in seen_sleepy_node()
262 static struct qnode *get_tail_qnode(struct qspinlock *lock, u32 val) in get_tail_qnode() argument
264 int cpu = decode_tail_cpu(val); in get_tail_qnode()
287 static __always_inline bool __yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt,… in __yield_to_locked_owner() argument
293 BUG_ON(!(val & _Q_LOCKED_VAL)); in __yield_to_locked_owner()
301 owner = get_owner_cpu(val); in __yield_to_locked_owner()
309 seen_sleepy_owner(lock, val); in __yield_to_locked_owner()
321 if (READ_ONCE(lock->val) == val) { in __yield_to_locked_owner()
340 static __always_inline bool yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt) in yield_to_locked_owner() argument
342 return __yield_to_locked_owner(lock, val, paravirt, false); in yield_to_locked_owner()
346 static __always_inline bool yield_head_to_locked_owner(struct qspinlock *lock, u32 val, bool paravi… in yield_head_to_locked_owner() argument
350 if ((val & _Q_MUST_Q_VAL) && pv_yield_allow_steal) in yield_head_to_locked_owner()
353 return __yield_to_locked_owner(lock, val, paravirt, mustq); in yield_head_to_locked_owner()
356 static __always_inline void propagate_yield_cpu(struct qnode *node, u32 val, int *set_yield_cpu, bo… in propagate_yield_cpu() argument
366 owner = get_owner_cpu(val); in propagate_yield_cpu()
384 static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode *node, u32 val, bool… in yield_to_prev() argument
386 int prev_cpu = decode_tail_cpu(val); in yield_to_prev()
409 if (get_owner_cpu(READ_ONCE(lock->val)) != yield_cpu) in yield_to_prev()
415 seen_sleepy_node(lock, val); in yield_to_prev()
439 seen_sleepy_node(lock, val); in yield_to_prev()
456 static __always_inline bool steal_break(u32 val, int iters, bool paravirt, bool sleepy) in steal_break() argument
463 int cpu = get_owner_cpu(val); in steal_break()
475 u32 val; in try_to_steal_lock() local
487 val = READ_ONCE(lock->val); in try_to_steal_lock()
488 if (val & _Q_MUST_Q_VAL) in try_to_steal_lock()
492 if (unlikely(!(val & _Q_LOCKED_VAL))) { in try_to_steal_lock()
498 preempted = yield_to_locked_owner(lock, val, paravirt); in try_to_steal_lock()
503 if (val & _Q_SLEEPY_VAL) { in try_to_steal_lock()
511 !(val & _Q_SLEEPY_VAL)) { in try_to_steal_lock()
512 if (try_set_sleepy(lock, val)) in try_to_steal_lock()
513 val |= _Q_SLEEPY_VAL; in try_to_steal_lock()
533 } while (!steal_break(val, iters, paravirt, sleepy)); in try_to_steal_lock()
544 u32 val, old, tail; in queued_spin_lock_mcs_queue() local
630 val = READ_ONCE(lock->val); in queued_spin_lock_mcs_queue()
631 if (!(val & _Q_LOCKED_VAL)) in queued_spin_lock_mcs_queue()
637 if (val & _Q_SLEEPY_VAL) { in queued_spin_lock_mcs_queue()
645 !(val & _Q_SLEEPY_VAL)) { in queued_spin_lock_mcs_queue()
646 if (try_set_sleepy(lock, val)) in queued_spin_lock_mcs_queue()
647 val |= _Q_SLEEPY_VAL; in queued_spin_lock_mcs_queue()
651 propagate_yield_cpu(node, val, &set_yield_cpu, paravirt); in queued_spin_lock_mcs_queue()
652 preempted = yield_head_to_locked_owner(lock, val, paravirt); in queued_spin_lock_mcs_queue()
671 val |= _Q_MUST_Q_VAL; in queued_spin_lock_mcs_queue()
751 static int steal_spins_set(void *data, u64 val) in steal_spins_set() argument
755 steal_spins = val; in steal_spins_set()
767 if (val && !steal_spins) { in steal_spins_set()
771 steal_spins = val; in steal_spins_set()
772 } else if (!val && steal_spins) { in steal_spins_set()
773 steal_spins = val; in steal_spins_set()
778 steal_spins = val; in steal_spins_set()
786 static int steal_spins_get(void *data, u64 *val) in steal_spins_get() argument
788 *val = steal_spins; in steal_spins_get()
795 static int remote_steal_spins_set(void *data, u64 val) in remote_steal_spins_set() argument
797 remote_steal_spins = val; in remote_steal_spins_set()
802 static int remote_steal_spins_get(void *data, u64 *val) in remote_steal_spins_get() argument
804 *val = remote_steal_spins; in remote_steal_spins_get()
811 static int head_spins_set(void *data, u64 val) in head_spins_set() argument
813 head_spins = val; in head_spins_set()
818 static int head_spins_get(void *data, u64 *val) in head_spins_get() argument
820 *val = head_spins; in head_spins_get()
827 static int pv_yield_owner_set(void *data, u64 val) in pv_yield_owner_set() argument
829 pv_yield_owner = !!val; in pv_yield_owner_set()
834 static int pv_yield_owner_get(void *data, u64 *val) in pv_yield_owner_get() argument
836 *val = pv_yield_owner; in pv_yield_owner_get()
843 static int pv_yield_allow_steal_set(void *data, u64 val) in pv_yield_allow_steal_set() argument
845 pv_yield_allow_steal = !!val; in pv_yield_allow_steal_set()
850 static int pv_yield_allow_steal_get(void *data, u64 *val) in pv_yield_allow_steal_get() argument
852 *val = pv_yield_allow_steal; in pv_yield_allow_steal_get()
859 static int pv_spin_on_preempted_owner_set(void *data, u64 val) in pv_spin_on_preempted_owner_set() argument
861 pv_spin_on_preempted_owner = !!val; in pv_spin_on_preempted_owner_set()
866 static int pv_spin_on_preempted_owner_get(void *data, u64 *val) in pv_spin_on_preempted_owner_get() argument
868 *val = pv_spin_on_preempted_owner; in pv_spin_on_preempted_owner_get()
875 static int pv_sleepy_lock_set(void *data, u64 val) in pv_sleepy_lock_set() argument
877 pv_sleepy_lock = !!val; in pv_sleepy_lock_set()
882 static int pv_sleepy_lock_get(void *data, u64 *val) in pv_sleepy_lock_get() argument
884 *val = pv_sleepy_lock; in pv_sleepy_lock_get()
891 static int pv_sleepy_lock_sticky_set(void *data, u64 val) in pv_sleepy_lock_sticky_set() argument
893 pv_sleepy_lock_sticky = !!val; in pv_sleepy_lock_sticky_set()
898 static int pv_sleepy_lock_sticky_get(void *data, u64 *val) in pv_sleepy_lock_sticky_get() argument
900 *val = pv_sleepy_lock_sticky; in pv_sleepy_lock_sticky_get()
907 static int pv_sleepy_lock_interval_ns_set(void *data, u64 val) in pv_sleepy_lock_interval_ns_set() argument
909 pv_sleepy_lock_interval_ns = val; in pv_sleepy_lock_interval_ns_set()
914 static int pv_sleepy_lock_interval_ns_get(void *data, u64 *val) in pv_sleepy_lock_interval_ns_get() argument
916 *val = pv_sleepy_lock_interval_ns; in pv_sleepy_lock_interval_ns_get()
923 static int pv_sleepy_lock_factor_set(void *data, u64 val) in pv_sleepy_lock_factor_set() argument
925 pv_sleepy_lock_factor = val; in pv_sleepy_lock_factor_set()
930 static int pv_sleepy_lock_factor_get(void *data, u64 *val) in pv_sleepy_lock_factor_get() argument
932 *val = pv_sleepy_lock_factor; in pv_sleepy_lock_factor_get()
939 static int pv_yield_prev_set(void *data, u64 val) in pv_yield_prev_set() argument
941 pv_yield_prev = !!val; in pv_yield_prev_set()
946 static int pv_yield_prev_get(void *data, u64 *val) in pv_yield_prev_get() argument
948 *val = pv_yield_prev; in pv_yield_prev_get()
955 static int pv_yield_propagate_owner_set(void *data, u64 val) in pv_yield_propagate_owner_set() argument
957 pv_yield_propagate_owner = !!val; in pv_yield_propagate_owner_set()
962 static int pv_yield_propagate_owner_get(void *data, u64 *val) in pv_yield_propagate_owner_get() argument
964 *val = pv_yield_propagate_owner; in pv_yield_propagate_owner_get()
971 static int pv_prod_head_set(void *data, u64 val) in pv_prod_head_set() argument
973 pv_prod_head = !!val; in pv_prod_head_set()
978 static int pv_prod_head_get(void *data, u64 *val) in pv_prod_head_get() argument
980 *val = pv_prod_head; in pv_prod_head_get()